From 73d62757b0a8547acdd2fbb49bbbb735a6308407 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Thu, 28 Nov 2024 20:12:27 +0300 Subject: [PATCH 001/249] JPEG XL support --- modules/api/helpers.py | 7 +++++++ modules/generation_parameters_copypaste.py | 2 ++ modules/gr_tempdir.py | 2 +- modules/images.py | 8 ++++++++ modules/loader.py | 3 +++ modules/shared.py | 4 ++-- modules/ui.py | 1 + modules/ui_extra_networks.py | 6 +++--- requirements.txt | 1 + 9 files changed, 28 insertions(+), 6 deletions(-) diff --git a/modules/api/helpers.py b/modules/api/helpers.py index 2d6ae8110..da826a81e 100644 --- a/modules/api/helpers.py +++ b/modules/api/helpers.py @@ -75,6 +75,13 @@ def save_image(image, fn, ext): image = image.point(lambda p: p * 0.0038910505836576).convert("RGB") exif_bytes = piexif.dump({ "Exif": { piexif.ExifIFD.UserComment: piexif.helper.UserComment.dump(parameters or "", encoding="unicode") } }) image.save(fn, format=image_format, quality=shared.opts.jpeg_quality, lossless=shared.opts.webp_lossless, exif=exif_bytes) + elif image_format == 'JXL': + if image.mode == 'I;16': + image = image.point(lambda p: p * 0.0038910505836576).convert("RGB") + elif image.mode not in {"RGB", "RGBA"}: + image = image.convert("RGBA") + exif_bytes = piexif.dump({ "Exif": { piexif.ExifIFD.UserComment: piexif.helper.UserComment.dump(parameters or "", encoding="unicode") } }) + image.save(fn, format=image_format, quality=shared.opts.jpeg_quality, lossless=shared.opts.webp_lossless, exif=exif_bytes) else: # shared.log.warning(f'Unrecognized image format: {extension} attempting save as {image_format}') image.save(fn, format=image_format, quality=shared.opts.jpeg_quality) diff --git a/modules/generation_parameters_copypaste.py b/modules/generation_parameters_copypaste.py index 36b5ea382..ea53c6b23 100644 --- a/modules/generation_parameters_copypaste.py +++ b/modules/generation_parameters_copypaste.py @@ -67,6 +67,8 @@ def image_from_url_text(filedata): filedata = filedata[len("data:image/webp;base64,"):] if filedata.startswith("data:image/jpeg;base64,"): filedata = filedata[len("data:image/jpeg;base64,"):] + if filedata.startswith("data:image/jxl;base64,"): + filedata = filedata[len("data:image/jxl;base64,"):] filedata = base64.decodebytes(filedata.encode('utf-8')) image = Image.open(io.BytesIO(filedata)) images.read_info_from_image(image) diff --git a/modules/gr_tempdir.py b/modules/gr_tempdir.py index 0ee15b314..9076ad407 100644 --- a/modules/gr_tempdir.py +++ b/modules/gr_tempdir.py @@ -93,7 +93,7 @@ def cleanup_tmpdr(): for root, _dirs, files in os.walk(temp_dir, topdown=False): for name in files: _, extension = os.path.splitext(name) - if extension != ".png" and extension != ".jpg" and extension != ".webp": + if extension not in {".png", ".jpg", ".webp", ".jxl"}: continue filename = os.path.join(root, name) os.remove(filename) diff --git a/modules/images.py b/modules/images.py index 910349bef..4a32d6d9c 100644 --- a/modules/images.py +++ b/modules/images.py @@ -70,6 +70,14 @@ def atomically_save_image(): save_args = { 'optimize': True, 'quality': shared.opts.jpeg_quality, 'lossless': shared.opts.webp_lossless } if shared.opts.image_metadata: save_args['exif'] = piexif.dump({ "Exif": { piexif.ExifIFD.UserComment: piexif.helper.UserComment.dump(exifinfo, encoding="unicode") } }) + elif image_format == 'JXL': + if image.mode == 'I;16': + image = image.point(lambda p: p * 0.0038910505836576).convert("RGB") + elif image.mode not in {"RGB", "RGBA"}: + image = image.convert("RGBA") + save_args = { 'optimize': True, 'quality': shared.opts.jpeg_quality, 'lossless': shared.opts.webp_lossless } + if shared.opts.image_metadata: + save_args['exif'] = piexif.dump({ "Exif": { piexif.ExifIFD.UserComment: piexif.helper.UserComment.dump(exifinfo, encoding="unicode") } }) else: save_args = { 'quality': shared.opts.jpeg_quality } try: diff --git a/modules/loader.py b/modules/loader.py index cd51cc8eb..ff7b34ee1 100644 --- a/modules/loader.py +++ b/modules/loader.py @@ -72,6 +72,9 @@ logging.getLogger("diffusers.loaders.single_file").setLevel(logging.ERROR) timer.startup.record("diffusers") +import pillow_jxl # pylint: disable=W0611,C0411 +from PIL import Image # pylint: disable=W0611,C0411 +timer.startup.record("pillow") # patch different progress bars import tqdm as tqdm_lib # pylint: disable=C0411 diff --git a/modules/shared.py b/modules/shared.py index 5b54a0de2..9ba98469e 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -679,7 +679,7 @@ def get_default_modes(): options_templates.update(options_section(('saving-images', "Image Options"), { "keep_incomplete": OptionInfo(True, "Keep incomplete images"), "samples_save": OptionInfo(True, "Save all generated images"), - "samples_format": OptionInfo('jpg', 'File format', gr.Dropdown, {"choices": ["jpg", "png", "webp", "tiff", "jp2"]}), + "samples_format": OptionInfo('jpg', 'File format', gr.Dropdown, {"choices": ["jpg", "png", "webp", "tiff", "jp2", "jxl"]}), "jpeg_quality": OptionInfo(90, "Image quality", gr.Slider, {"minimum": 1, "maximum": 100, "step": 1}), "img_max_size_mp": OptionInfo(1000, "Maximum image size (MP)", gr.Slider, {"minimum": 100, "maximum": 2000, "step": 1}), "webp_lossless": OptionInfo(False, "WebP lossless compression"), @@ -694,7 +694,7 @@ def get_default_modes(): "save_log_fn": OptionInfo("", "Append image info JSON file", component_args=hide_dirs), "image_sep_grid": OptionInfo("

Grid Options

", "", gr.HTML), "grid_save": OptionInfo(True, "Save all generated image grids"), - "grid_format": OptionInfo('jpg', 'File format', gr.Dropdown, {"choices": ["jpg", "png", "webp", "tiff", "jp2"]}), + "grid_format": OptionInfo('jpg', 'File format', gr.Dropdown, {"choices": ["jpg", "png", "webp", "tiff", "jp2", "jxl"]}), "n_rows": OptionInfo(-1, "Row count", gr.Slider, {"minimum": -1, "maximum": 16, "step": 1}), "grid_background": OptionInfo("#000000", "Grid background color", gr.ColorPicker, {}), "font": OptionInfo("", "Font file"), diff --git a/modules/ui.py b/modules/ui.py index 4490bbf8c..7f72c716e 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -16,6 +16,7 @@ mimetypes.init() mimetypes.add_type('application/javascript', '.js') mimetypes.add_type('image/webp', '.webp') +mimetypes.add_type('image/jxl', '.jxl') log = shared.log opts = shared.opts cmd_opts = shared.cmd_opts diff --git a/modules/ui_extra_networks.py b/modules/ui_extra_networks.py index c326219df..1365e1c24 100644 --- a/modules/ui_extra_networks.py +++ b/modules/ui_extra_networks.py @@ -331,7 +331,7 @@ def find_preview_file(self, path): return 'html/card-no-preview.png' if os.path.join('models', 'Reference') in path: return path - exts = ["jpg", "jpeg", "png", "webp", "tiff", "jp2"] + exts = ["jpg", "jpeg", "png", "webp", "tiff", "jp2", "jxl"] reference_path = os.path.abspath(os.path.join('models', 'Reference')) files = list(files_cache.list_files(reference_path, ext_filter=exts, recursive=False)) if shared.opts.diffusers_dir in path: @@ -360,7 +360,7 @@ def update_all_previews(self, items): t0 = time.time() reference_path = os.path.abspath(os.path.join('models', 'Reference')) possible_paths = list(set([os.path.dirname(item['filename']) for item in items] + [reference_path])) - exts = ["jpg", "jpeg", "png", "webp", "tiff", "jp2"] + exts = ["jpg", "jpeg", "png", "webp", "tiff", "jp2", "jxl"] all_previews = list(files_cache.list_files(*possible_paths, ext_filter=exts, recursive=False)) all_previews_fn = [os.path.basename(x) for x in all_previews] for item in items: @@ -680,7 +680,7 @@ def fn_save_img(image): return image def fn_delete_img(_image): - preview_extensions = ["jpg", "jpeg", "png", "webp", "tiff", "jp2"] + preview_extensions = ["jpg", "jpeg", "png", "webp", "tiff", "jp2", "jxl"] fn = os.path.splitext(ui.last_item.filename)[0] for file in [f'{fn}{mid}{ext}' for ext in preview_extensions for mid in ['.thumb.', '.preview.', '.']]: if os.path.exists(file): diff --git a/requirements.txt b/requirements.txt index 12a9f85cb..68c64954d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -55,6 +55,7 @@ tokenizers==0.20.3 transformers==4.46.2 urllib3==1.26.19 Pillow==10.4.0 +pillow-jxl-plugin==1.3.0 timm==0.9.16 pydantic==1.10.15 pyparsing==3.1.4 From a43ebc33ce5ffebc991ce8fe4bfad4224a730108 Mon Sep 17 00:00:00 2001 From: AI-Casanova <54461896+AI-Casanova@users.noreply.github.com> Date: Sat, 23 Nov 2024 21:57:03 -0600 Subject: [PATCH 002/249] Major lora refactor: works on my machine edition --- .../Lora/scripts/lora_script.py | 14 +- modules/lora/extra_networks_lora.py | 151 ++++++ modules/lora/lora.py | 8 + modules/lora/lora_convert.py | 477 ++++++++++++++++++ modules/lora/lora_extract.py | 271 ++++++++++ modules/lora/lyco_helpers.py | 66 +++ modules/lora/network.py | 187 +++++++ modules/lora/network_full.py | 26 + modules/lora/network_glora.py | 30 ++ modules/lora/network_hada.py | 46 ++ modules/lora/network_ia3.py | 24 + modules/lora/network_lokr.py | 57 +++ modules/lora/network_lora.py | 78 +++ modules/lora/network_norm.py | 23 + modules/lora/network_oft.py | 81 +++ modules/lora/network_overrides.py | 49 ++ modules/lora/networks.py | 453 +++++++++++++++++ modules/lora/ui_extra_networks_lora.py | 123 +++++ modules/processing_diffusers.py | 5 + modules/shared.py | 1 + scripts/lora_script.py | 62 +++ 21 files changed, 2225 insertions(+), 7 deletions(-) create mode 100644 modules/lora/extra_networks_lora.py create mode 100644 modules/lora/lora.py create mode 100644 modules/lora/lora_convert.py create mode 100644 modules/lora/lora_extract.py create mode 100644 modules/lora/lyco_helpers.py create mode 100644 modules/lora/network.py create mode 100644 modules/lora/network_full.py create mode 100644 modules/lora/network_glora.py create mode 100644 modules/lora/network_hada.py create mode 100644 modules/lora/network_ia3.py create mode 100644 modules/lora/network_lokr.py create mode 100644 modules/lora/network_lora.py create mode 100644 modules/lora/network_norm.py create mode 100644 modules/lora/network_oft.py create mode 100644 modules/lora/network_overrides.py create mode 100644 modules/lora/networks.py create mode 100644 modules/lora/ui_extra_networks_lora.py create mode 100644 scripts/lora_script.py diff --git a/extensions-builtin/Lora/scripts/lora_script.py b/extensions-builtin/Lora/scripts/lora_script.py index ffbef47d9..dea2985b3 100644 --- a/extensions-builtin/Lora/scripts/lora_script.py +++ b/extensions-builtin/Lora/scripts/lora_script.py @@ -5,7 +5,7 @@ from network import NetworkOnDisk from ui_extra_networks_lora import ExtraNetworksPageLora from extra_networks_lora import ExtraNetworkLora -from modules import script_callbacks, extra_networks, ui_extra_networks, ui_models # pylint: disable=unused-import +from modules import script_callbacks, extra_networks, ui_extra_networks, ui_models, shared # pylint: disable=unused-import re_lora = re.compile(" 0 else 1.0 + v = np.interp(step, m[1], m[0]) + return v + else: + return m + + stepwise = calculate_weight(sorted_positions(param), step, steps) + return stepwise + + +def prompt(p): + if shared.opts.lora_apply_tags == 0: + return + all_tags = [] + for loaded in networks.loaded_networks: + page = [en for en in shared.extra_networks if en.name == 'lora'][0] + item = page.create_item(loaded.name) + tags = (item or {}).get("tags", {}) + loaded.tags = list(tags) + if len(loaded.tags) == 0: + loaded.tags.append(loaded.name) + if shared.opts.lora_apply_tags > 0: + loaded.tags = loaded.tags[:shared.opts.lora_apply_tags] + all_tags.extend(loaded.tags) + if len(all_tags) > 0: + shared.log.debug(f"Load network: type=LoRA tags={all_tags} max={shared.opts.lora_apply_tags} apply") + all_tags = ', '.join(all_tags) + p.extra_generation_params["LoRA tags"] = all_tags + if '_tags_' in p.prompt: + p.prompt = p.prompt.replace('_tags_', all_tags) + else: + p.prompt = f"{p.prompt}, {all_tags}" + if p.all_prompts is not None: + for i in range(len(p.all_prompts)): + if '_tags_' in p.all_prompts[i]: + p.all_prompts[i] = p.all_prompts[i].replace('_tags_', all_tags) + else: + p.all_prompts[i] = f"{p.all_prompts[i]}, {all_tags}" + + +def infotext(p): + names = [i.name for i in networks.loaded_networks] + if len(names) > 0: + p.extra_generation_params["LoRA networks"] = ", ".join(names) + if shared.opts.lora_add_hashes_to_infotext: + network_hashes = [] + for item in networks.loaded_networks: + if not item.network_on_disk.shorthash: + continue + network_hashes.append(item.network_on_disk.shorthash) + if len(network_hashes) > 0: + p.extra_generation_params["LoRA hashes"] = ", ".join(network_hashes) + + +def parse(p, params_list, step=0): + names = [] + te_multipliers = [] + unet_multipliers = [] + dyn_dims = [] + for params in params_list: + assert params.items + names.append(params.positional[0]) + te_multiplier = params.named.get("te", params.positional[1] if len(params.positional) > 1 else shared.opts.extra_networks_default_multiplier) + if isinstance(te_multiplier, str) and "@" in te_multiplier: + te_multiplier = get_stepwise(te_multiplier, step, p.steps) + else: + te_multiplier = float(te_multiplier) + unet_multiplier = [params.positional[2] if len(params.positional) > 2 else te_multiplier] * 3 + unet_multiplier = [params.named.get("unet", unet_multiplier[0])] * 3 + unet_multiplier[0] = params.named.get("in", unet_multiplier[0]) + unet_multiplier[1] = params.named.get("mid", unet_multiplier[1]) + unet_multiplier[2] = params.named.get("out", unet_multiplier[2]) + for i in range(len(unet_multiplier)): + if isinstance(unet_multiplier[i], str) and "@" in unet_multiplier[i]: + unet_multiplier[i] = get_stepwise(unet_multiplier[i], step, p.steps) + else: + unet_multiplier[i] = float(unet_multiplier[i]) + dyn_dim = int(params.positional[3]) if len(params.positional) > 3 else None + dyn_dim = int(params.named["dyn"]) if "dyn" in params.named else dyn_dim + te_multipliers.append(te_multiplier) + unet_multipliers.append(unet_multiplier) + dyn_dims.append(dyn_dim) + return names, te_multipliers, unet_multipliers, dyn_dims + + +class ExtraNetworkLora(extra_networks.ExtraNetwork): + + def __init__(self): + super().__init__('lora') + self.active = False + self.model = None + self.errors = {} + + def activate(self, p, params_list, step=0): + t0 = time.time() + self.errors.clear() + if self.active: + if self.model != shared.opts.sd_model_checkpoint: # reset if model changed + self.active = False + if len(params_list) > 0 and not self.active: # activate patches once + shared.log.debug(f'Activate network: type=LoRA model="{shared.opts.sd_model_checkpoint}"') + self.active = True + self.model = shared.opts.sd_model_checkpoint + names, te_multipliers, unet_multipliers, dyn_dims = parse(p, params_list, step) + networks.load_networks(names, te_multipliers, unet_multipliers, dyn_dims) + t1 = time.time() + if len(networks.loaded_networks) > 0 and step == 0: + infotext(p) + prompt(p) + shared.log.info(f'Load network: type=LoRA apply={[n.name for n in networks.loaded_networks]} te={te_multipliers} unet={unet_multipliers} dims={dyn_dims} load={t1-t0:.2f}') + + def deactivate(self, p): + t0 = time.time() + if shared.native and len(networks.diffuser_loaded) > 0: + if hasattr(shared.sd_model, "unload_lora_weights") and hasattr(shared.sd_model, "text_encoder"): + if not (shared.compiled_model_state is not None and shared.compiled_model_state.is_compiled is True): + try: + if shared.opts.lora_fuse_diffusers: + shared.sd_model.unfuse_lora() + shared.sd_model.unload_lora_weights() # fails for non-CLIP models + except Exception: + pass + t1 = time.time() + networks.timer['restore'] += t1 - t0 + if self.active and networks.debug: + shared.log.debug(f"Network end: type=LoRA load={networks.timer['load']:.2f} apply={networks.timer['apply']:.2f} restore={networks.timer['restore']:.2f}") + if self.errors: + for k, v in self.errors.items(): + shared.log.error(f'LoRA: name="{k}" errors={v}') + self.errors.clear() diff --git a/modules/lora/lora.py b/modules/lora/lora.py new file mode 100644 index 000000000..33adfe05c --- /dev/null +++ b/modules/lora/lora.py @@ -0,0 +1,8 @@ +# import networks +# +# list_available_loras = networks.list_available_networks +# available_loras = networks.available_networks +# available_lora_aliases = networks.available_network_aliases +# available_lora_hash_lookup = networks.available_network_hash_lookup +# forbidden_lora_aliases = networks.forbidden_network_aliases +# loaded_loras = networks.loaded_networks diff --git a/modules/lora/lora_convert.py b/modules/lora/lora_convert.py new file mode 100644 index 000000000..6bf563125 --- /dev/null +++ b/modules/lora/lora_convert.py @@ -0,0 +1,477 @@ +import os +import re +import bisect +from typing import Dict +import torch +from modules import shared + + +debug = os.environ.get('SD_LORA_DEBUG', None) is not None +suffix_conversion = { + "attentions": {}, + "resnets": { + "conv1": "in_layers_2", + "conv2": "out_layers_3", + "norm1": "in_layers_0", + "norm2": "out_layers_0", + "time_emb_proj": "emb_layers_1", + "conv_shortcut": "skip_connection", + } +} +re_digits = re.compile(r"\d+") +re_x_proj = re.compile(r"(.*)_([qkv]_proj)$") +re_compiled = {} + + +def make_unet_conversion_map() -> Dict[str, str]: + unet_conversion_map_layer = [] + + for i in range(3): # num_blocks is 3 in sdxl + # loop over downblocks/upblocks + for j in range(2): + # loop over resnets/attentions for downblocks + hf_down_res_prefix = f"down_blocks.{i}.resnets.{j}." + sd_down_res_prefix = f"input_blocks.{3 * i + j + 1}.0." + unet_conversion_map_layer.append((sd_down_res_prefix, hf_down_res_prefix)) + if i < 3: + # no attention layers in down_blocks.3 + hf_down_atn_prefix = f"down_blocks.{i}.attentions.{j}." + sd_down_atn_prefix = f"input_blocks.{3 * i + j + 1}.1." + unet_conversion_map_layer.append((sd_down_atn_prefix, hf_down_atn_prefix)) + + for j in range(3): + # loop over resnets/attentions for upblocks + hf_up_res_prefix = f"up_blocks.{i}.resnets.{j}." + sd_up_res_prefix = f"output_blocks.{3 * i + j}.0." + unet_conversion_map_layer.append((sd_up_res_prefix, hf_up_res_prefix)) + # if i > 0: commentout for sdxl + # no attention layers in up_blocks.0 + hf_up_atn_prefix = f"up_blocks.{i}.attentions.{j}." + sd_up_atn_prefix = f"output_blocks.{3 * i + j}.1." + unet_conversion_map_layer.append((sd_up_atn_prefix, hf_up_atn_prefix)) + + if i < 3: + # no downsample in down_blocks.3 + hf_downsample_prefix = f"down_blocks.{i}.downsamplers.0.conv." + sd_downsample_prefix = f"input_blocks.{3 * (i + 1)}.0.op." + unet_conversion_map_layer.append((sd_downsample_prefix, hf_downsample_prefix)) + # no upsample in up_blocks.3 + hf_upsample_prefix = f"up_blocks.{i}.upsamplers.0." + sd_upsample_prefix = f"output_blocks.{3 * i + 2}.{2}." # change for sdxl + unet_conversion_map_layer.append((sd_upsample_prefix, hf_upsample_prefix)) + + hf_mid_atn_prefix = "mid_block.attentions.0." + sd_mid_atn_prefix = "middle_block.1." + unet_conversion_map_layer.append((sd_mid_atn_prefix, hf_mid_atn_prefix)) + + for j in range(2): + hf_mid_res_prefix = f"mid_block.resnets.{j}." + sd_mid_res_prefix = f"middle_block.{2 * j}." + unet_conversion_map_layer.append((sd_mid_res_prefix, hf_mid_res_prefix)) + + unet_conversion_map_resnet = [ + # (stable-diffusion, HF Diffusers) + ("in_layers.0.", "norm1."), + ("in_layers.2.", "conv1."), + ("out_layers.0.", "norm2."), + ("out_layers.3.", "conv2."), + ("emb_layers.1.", "time_emb_proj."), + ("skip_connection.", "conv_shortcut."), + ] + + unet_conversion_map = [] + for sd, hf in unet_conversion_map_layer: + if "resnets" in hf: + for sd_res, hf_res in unet_conversion_map_resnet: + unet_conversion_map.append((sd + sd_res, hf + hf_res)) + else: + unet_conversion_map.append((sd, hf)) + + for j in range(2): + hf_time_embed_prefix = f"time_embedding.linear_{j + 1}." + sd_time_embed_prefix = f"time_embed.{j * 2}." + unet_conversion_map.append((sd_time_embed_prefix, hf_time_embed_prefix)) + + for j in range(2): + hf_label_embed_prefix = f"add_embedding.linear_{j + 1}." + sd_label_embed_prefix = f"label_emb.0.{j * 2}." + unet_conversion_map.append((sd_label_embed_prefix, hf_label_embed_prefix)) + + unet_conversion_map.append(("input_blocks.0.0.", "conv_in.")) + unet_conversion_map.append(("out.0.", "conv_norm_out.")) + unet_conversion_map.append(("out.2.", "conv_out.")) + + sd_hf_conversion_map = {sd.replace(".", "_")[:-1]: hf.replace(".", "_")[:-1] for sd, hf in unet_conversion_map} + return sd_hf_conversion_map + + +class KeyConvert: + def __init__(self): + self.is_sdxl = True if shared.sd_model_type == "sdxl" else False + self.UNET_CONVERSION_MAP = make_unet_conversion_map() if self.is_sdxl else None + self.LORA_PREFIX_UNET = "lora_unet_" + self.LORA_PREFIX_TEXT_ENCODER = "lora_te_" + self.OFT_PREFIX_UNET = "oft_unet_" + # SDXL: must starts with LORA_PREFIX_TEXT_ENCODER + self.LORA_PREFIX_TEXT_ENCODER1 = "lora_te1_" + self.LORA_PREFIX_TEXT_ENCODER2 = "lora_te2_" + + def __call__(self, key): + if self.is_sdxl: + if "diffusion_model" in key: # Fix NTC Slider naming error + key = key.replace("diffusion_model", "lora_unet") + map_keys = list(self.UNET_CONVERSION_MAP.keys()) # prefix of U-Net modules + map_keys.sort() + search_key = key.replace(self.LORA_PREFIX_UNET, "").replace(self.OFT_PREFIX_UNET, "").replace(self.LORA_PREFIX_TEXT_ENCODER1, "").replace(self.LORA_PREFIX_TEXT_ENCODER2, "") + position = bisect.bisect_right(map_keys, search_key) + map_key = map_keys[position - 1] + if search_key.startswith(map_key): + key = key.replace(map_key, self.UNET_CONVERSION_MAP[map_key]).replace("oft", "lora") # pylint: disable=unsubscriptable-object + if "lycoris" in key and "transformer" in key: + key = key.replace("lycoris", "lora_transformer") + sd_module = shared.sd_model.network_layer_mapping.get(key, None) + if sd_module is None: + sd_module = shared.sd_model.network_layer_mapping.get(key.replace("guidance", "timestep"), None) # FLUX1 fix + if debug and sd_module is None: + raise RuntimeError(f"LoRA key not found in network_layer_mapping: key={key} mapping={shared.sd_model.network_layer_mapping.keys()}") + return key, sd_module + + +# Taken from https://github.com/huggingface/diffusers/blob/main/src/diffusers/loaders/lora_conversion_utils.py +# Modified from 'lora_A' and 'lora_B' to 'lora_down' and 'lora_up' +# Added early exit +# The utilities under `_convert_kohya_flux_lora_to_diffusers()` +# are taken from https://github.com/kohya-ss/sd-scripts/blob/a61cf73a5cb5209c3f4d1a3688dd276a4dfd1ecb/networks/convert_flux_lora.py +# All credits go to `kohya-ss`. +def _convert_to_ai_toolkit(sds_sd, ait_sd, sds_key, ait_key): + if sds_key + ".lora_down.weight" not in sds_sd: + return + down_weight = sds_sd.pop(sds_key + ".lora_down.weight") + + # scale weight by alpha and dim + rank = down_weight.shape[0] + alpha = sds_sd.pop(sds_key + ".alpha").item() # alpha is scalar + scale = alpha / rank # LoRA is scaled by 'alpha / rank' in forward pass, so we need to scale it back here + + # calculate scale_down and scale_up to keep the same value. if scale is 4, scale_down is 2 and scale_up is 2 + scale_down = scale + scale_up = 1.0 + while scale_down * 2 < scale_up: + scale_down *= 2 + scale_up /= 2 + + ait_sd[ait_key + ".lora_down.weight"] = down_weight * scale_down + ait_sd[ait_key + ".lora_up.weight"] = sds_sd.pop(sds_key + ".lora_up.weight") * scale_up + +def _convert_to_ai_toolkit_cat(sds_sd, ait_sd, sds_key, ait_keys, dims=None): + if sds_key + ".lora_down.weight" not in sds_sd: + return + down_weight = sds_sd.pop(sds_key + ".lora_down.weight") + up_weight = sds_sd.pop(sds_key + ".lora_up.weight") + sd_lora_rank = down_weight.shape[0] + + # scale weight by alpha and dim + alpha = sds_sd.pop(sds_key + ".alpha") + scale = alpha / sd_lora_rank + + # calculate scale_down and scale_up + scale_down = scale + scale_up = 1.0 + while scale_down * 2 < scale_up: + scale_down *= 2 + scale_up /= 2 + + down_weight = down_weight * scale_down + up_weight = up_weight * scale_up + + # calculate dims if not provided + num_splits = len(ait_keys) + if dims is None: + dims = [up_weight.shape[0] // num_splits] * num_splits + else: + assert sum(dims) == up_weight.shape[0] + + # check upweight is sparse or not + is_sparse = False + if sd_lora_rank % num_splits == 0: + ait_rank = sd_lora_rank // num_splits + is_sparse = True + i = 0 + for j in range(len(dims)): + for k in range(len(dims)): + if j == k: + continue + is_sparse = is_sparse and torch.all( + up_weight[i : i + dims[j], k * ait_rank : (k + 1) * ait_rank] == 0 + ) + i += dims[j] + # if is_sparse: + # print(f"weight is sparse: {sds_key}") + + # make ai-toolkit weight + ait_down_keys = [k + ".lora_down.weight" for k in ait_keys] + ait_up_keys = [k + ".lora_up.weight" for k in ait_keys] + if not is_sparse: + # down_weight is copied to each split + ait_sd.update({k: down_weight for k in ait_down_keys}) + + # up_weight is split to each split + ait_sd.update({k: v for k, v in zip(ait_up_keys, torch.split(up_weight, dims, dim=0))}) # noqa: C416 # pylint: disable=unnecessary-comprehension + else: + # down_weight is chunked to each split + ait_sd.update({k: v for k, v in zip(ait_down_keys, torch.chunk(down_weight, num_splits, dim=0))}) # noqa: C416 # pylint: disable=unnecessary-comprehension + + # up_weight is sparse: only non-zero values are copied to each split + i = 0 + for j in range(len(dims)): + ait_sd[ait_up_keys[j]] = up_weight[i : i + dims[j], j * ait_rank : (j + 1) * ait_rank].contiguous() + i += dims[j] + +def _convert_text_encoder_lora_key(key, lora_name): + """ + Converts a text encoder LoRA key to a Diffusers compatible key. + """ + if lora_name.startswith(("lora_te_", "lora_te1_")): + key_to_replace = "lora_te_" if lora_name.startswith("lora_te_") else "lora_te1_" + else: + key_to_replace = "lora_te2_" + + diffusers_name = key.replace(key_to_replace, "").replace("_", ".") + diffusers_name = diffusers_name.replace("text.model", "text_model") + diffusers_name = diffusers_name.replace("self.attn", "self_attn") + diffusers_name = diffusers_name.replace("q.proj.lora", "to_q_lora") + diffusers_name = diffusers_name.replace("k.proj.lora", "to_k_lora") + diffusers_name = diffusers_name.replace("v.proj.lora", "to_v_lora") + diffusers_name = diffusers_name.replace("out.proj.lora", "to_out_lora") + diffusers_name = diffusers_name.replace("text.projection", "text_projection") + + if "self_attn" in diffusers_name or "text_projection" in diffusers_name: + pass + elif "mlp" in diffusers_name: + # Be aware that this is the new diffusers convention and the rest of the code might + # not utilize it yet. + diffusers_name = diffusers_name.replace(".lora.", ".lora_linear_layer.") + return diffusers_name + +def _convert_kohya_flux_lora_to_diffusers(state_dict): + def _convert_sd_scripts_to_ai_toolkit(sds_sd): + ait_sd = {} + for i in range(19): + _convert_to_ai_toolkit( + sds_sd, + ait_sd, + f"lora_unet_double_blocks_{i}_img_attn_proj", + f"transformer.transformer_blocks.{i}.attn.to_out.0", + ) + _convert_to_ai_toolkit_cat( + sds_sd, + ait_sd, + f"lora_unet_double_blocks_{i}_img_attn_qkv", + [ + f"transformer.transformer_blocks.{i}.attn.to_q", + f"transformer.transformer_blocks.{i}.attn.to_k", + f"transformer.transformer_blocks.{i}.attn.to_v", + ], + ) + _convert_to_ai_toolkit( + sds_sd, + ait_sd, + f"lora_unet_double_blocks_{i}_img_mlp_0", + f"transformer.transformer_blocks.{i}.ff.net.0.proj", + ) + _convert_to_ai_toolkit( + sds_sd, + ait_sd, + f"lora_unet_double_blocks_{i}_img_mlp_2", + f"transformer.transformer_blocks.{i}.ff.net.2", + ) + _convert_to_ai_toolkit( + sds_sd, + ait_sd, + f"lora_unet_double_blocks_{i}_img_mod_lin", + f"transformer.transformer_blocks.{i}.norm1.linear", + ) + _convert_to_ai_toolkit( + sds_sd, + ait_sd, + f"lora_unet_double_blocks_{i}_txt_attn_proj", + f"transformer.transformer_blocks.{i}.attn.to_add_out", + ) + _convert_to_ai_toolkit_cat( + sds_sd, + ait_sd, + f"lora_unet_double_blocks_{i}_txt_attn_qkv", + [ + f"transformer.transformer_blocks.{i}.attn.add_q_proj", + f"transformer.transformer_blocks.{i}.attn.add_k_proj", + f"transformer.transformer_blocks.{i}.attn.add_v_proj", + ], + ) + _convert_to_ai_toolkit( + sds_sd, + ait_sd, + f"lora_unet_double_blocks_{i}_txt_mlp_0", + f"transformer.transformer_blocks.{i}.ff_context.net.0.proj", + ) + _convert_to_ai_toolkit( + sds_sd, + ait_sd, + f"lora_unet_double_blocks_{i}_txt_mlp_2", + f"transformer.transformer_blocks.{i}.ff_context.net.2", + ) + _convert_to_ai_toolkit( + sds_sd, + ait_sd, + f"lora_unet_double_blocks_{i}_txt_mod_lin", + f"transformer.transformer_blocks.{i}.norm1_context.linear", + ) + + for i in range(38): + _convert_to_ai_toolkit_cat( + sds_sd, + ait_sd, + f"lora_unet_single_blocks_{i}_linear1", + [ + f"transformer.single_transformer_blocks.{i}.attn.to_q", + f"transformer.single_transformer_blocks.{i}.attn.to_k", + f"transformer.single_transformer_blocks.{i}.attn.to_v", + f"transformer.single_transformer_blocks.{i}.proj_mlp", + ], + dims=[3072, 3072, 3072, 12288], + ) + _convert_to_ai_toolkit( + sds_sd, + ait_sd, + f"lora_unet_single_blocks_{i}_linear2", + f"transformer.single_transformer_blocks.{i}.proj_out", + ) + _convert_to_ai_toolkit( + sds_sd, + ait_sd, + f"lora_unet_single_blocks_{i}_modulation_lin", + f"transformer.single_transformer_blocks.{i}.norm.linear", + ) + + if len(sds_sd) > 0: + return None + + return ait_sd + + return _convert_sd_scripts_to_ai_toolkit(state_dict) + +def _convert_kohya_sd3_lora_to_diffusers(state_dict): + def _convert_sd_scripts_to_ai_toolkit(sds_sd): + ait_sd = {} + for i in range(38): + _convert_to_ai_toolkit_cat( + sds_sd, + ait_sd, + f"lora_unet_joint_blocks_{i}_context_block_attn_qkv", + [ + f"transformer.transformer_blocks.{i}.attn.to_q", + f"transformer.transformer_blocks.{i}.attn.to_k", + f"transformer.transformer_blocks.{i}.attn.to_v", + ], + ) + _convert_to_ai_toolkit( + sds_sd, + ait_sd, + f"lora_unet_joint_blocks_{i}_context_block_mlp_fc1", + f"transformer.transformer_blocks.{i}.ff_context.net.0.proj", + ) + _convert_to_ai_toolkit( + sds_sd, + ait_sd, + f"lora_unet_joint_blocks_{i}_context_block_mlp_fc2", + f"transformer.transformer_blocks.{i}.ff_context.net.2", + ) + _convert_to_ai_toolkit( + sds_sd, + ait_sd, + f"lora_unet_joint_blocks_{i}_x_block_mlp_fc1", + f"transformer.transformer_blocks.{i}.ff.net.0.proj", + ) + _convert_to_ai_toolkit( + sds_sd, + ait_sd, + f"lora_unet_joint_blocks_{i}_x_block_mlp_fc2", + f"transformer.transformer_blocks.{i}.ff.net.2", + ) + _convert_to_ai_toolkit( + sds_sd, + ait_sd, + f"lora_unet_joint_blocks_{i}_context_block_adaLN_modulation_1", + f"transformer.transformer_blocks.{i}.norm1_context.linear", + ) + _convert_to_ai_toolkit( + sds_sd, + ait_sd, + f"lora_unet_joint_blocks_{i}_x_block_adaLN_modulation_1", + f"transformer.transformer_blocks.{i}.norm1.linear", + ) + _convert_to_ai_toolkit( + sds_sd, + ait_sd, + f"lora_unet_joint_blocks_{i}_context_block_attn_proj", + f"transformer.transformer_blocks.{i}.attn.to_add_out", + ) + _convert_to_ai_toolkit( + sds_sd, + ait_sd, + f"lora_unet_joint_blocks_{i}_x_block_attn_proj", + f"transformer.transformer_blocks.{i}.attn.to_out_0", + ) + + _convert_to_ai_toolkit_cat( + sds_sd, + ait_sd, + f"lora_unet_joint_blocks_{i}_x_block_attn_qkv", + [ + f"transformer.transformer_blocks.{i}.attn.add_q_proj", + f"transformer.transformer_blocks.{i}.attn.add_k_proj", + f"transformer.transformer_blocks.{i}.attn.add_v_proj", + ], + ) + remaining_keys = list(sds_sd.keys()) + te_state_dict = {} + if remaining_keys: + if not all(k.startswith("lora_te1") for k in remaining_keys): + raise ValueError(f"Incompatible keys detected: \n\n {', '.join(remaining_keys)}") + for key in remaining_keys: + if not key.endswith("lora_down.weight"): + continue + + lora_name = key.split(".")[0] + lora_name_up = f"{lora_name}.lora_up.weight" + lora_name_alpha = f"{lora_name}.alpha" + diffusers_name = _convert_text_encoder_lora_key(key, lora_name) + + if lora_name.startswith(("lora_te_", "lora_te1_")): + down_weight = sds_sd.pop(key) + sd_lora_rank = down_weight.shape[0] + te_state_dict[diffusers_name] = down_weight + te_state_dict[diffusers_name.replace(".down.", ".up.")] = sds_sd.pop(lora_name_up) + + if lora_name_alpha in sds_sd: + alpha = sds_sd.pop(lora_name_alpha).item() + scale = alpha / sd_lora_rank + + scale_down = scale + scale_up = 1.0 + while scale_down * 2 < scale_up: + scale_down *= 2 + scale_up /= 2 + + te_state_dict[diffusers_name] *= scale_down + te_state_dict[diffusers_name.replace(".down.", ".up.")] *= scale_up + + if len(sds_sd) > 0: + print(f"Unsupported keys for ai-toolkit: {sds_sd.keys()}") + + if te_state_dict: + te_state_dict = {f"text_encoder.{module_name}": params for module_name, params in te_state_dict.items()} + + new_state_dict = {**ait_sd, **te_state_dict} + return new_state_dict + + return _convert_sd_scripts_to_ai_toolkit(state_dict) diff --git a/modules/lora/lora_extract.py b/modules/lora/lora_extract.py new file mode 100644 index 000000000..c2e0a275b --- /dev/null +++ b/modules/lora/lora_extract.py @@ -0,0 +1,271 @@ +import os +import time +import json +import datetime +import torch +from safetensors.torch import save_file +import gradio as gr +from rich import progress as p +from modules import shared, devices +from modules.ui_common import create_refresh_button +from modules.call_queue import wrap_gradio_gpu_call + + +class SVDHandler: + def __init__(self, maxrank=0, rank_ratio=1): + self.network_name: str = None + self.U: torch.Tensor = None + self.S: torch.Tensor = None + self.Vh: torch.Tensor = None + self.maxrank: int = maxrank + self.rank_ratio: float = rank_ratio + self.rank: int = 0 + self.out_size: int = None + self.in_size: int = None + self.kernel_size: tuple[int, int] = None + self.conv2d: bool = False + + def decompose(self, weight, backupweight): + self.conv2d = len(weight.size()) == 4 + self.kernel_size = None if not self.conv2d else weight.size()[2:4] + self.out_size, self.in_size = weight.size()[0:2] + diffweight = weight.clone().to(devices.device) + diffweight -= backupweight.to(devices.device) + if self.conv2d: + if self.conv2d and self.kernel_size != (1, 1): + diffweight = diffweight.flatten(start_dim=1) + else: + diffweight = diffweight.squeeze() + self.U, self.S, self.Vh = torch.svd_lowrank(diffweight.to(device=devices.device, dtype=torch.float), self.maxrank, 2) + # del diffweight + self.U = self.U.to(device=devices.cpu, dtype=torch.bfloat16) + self.S = self.S.to(device=devices.cpu, dtype=torch.bfloat16) + self.Vh = self.Vh.t().to(device=devices.cpu, dtype=torch.bfloat16) # svd_lowrank outputs a transposed matrix + + def findrank(self): + if self.rank_ratio < 1: + S_squared = self.S.pow(2) + S_fro_sq = float(torch.sum(S_squared)) + sum_S_squared = torch.cumsum(S_squared, dim=0) / S_fro_sq + index = int(torch.searchsorted(sum_S_squared, self.rank_ratio ** 2)) + 1 + index = max(1, min(index, len(self.S) - 1)) + self.rank = index + if self.maxrank > 0: + self.rank = min(self.rank, self.maxrank) + else: + self.rank = min(self.in_size, self.out_size, self.maxrank) + + def makeweights(self): + self.findrank() + up = self.U[:, :self.rank] @ torch.diag(self.S[:self.rank]) + down = self.Vh[:self.rank, :] + if self.conv2d and self.kernel_size is not None: + up = up.reshape(self.out_size, self.rank, 1, 1) + down = down.reshape(self.rank, self.in_size, self.kernel_size[0], self.kernel_size[1]) # pylint: disable=unsubscriptable-object + return_dict = {f'{self.network_name}.lora_up.weight': up.contiguous(), + f'{self.network_name}.lora_down.weight': down.contiguous(), + f'{self.network_name}.alpha': torch.tensor(down.shape[0]), + } + return return_dict + + +def loaded_lora(): + if not shared.sd_loaded: + return "" + loaded = set() + if hasattr(shared.sd_model, 'unet'): + for _name, module in shared.sd_model.unet.named_modules(): + current = getattr(module, "network_current_names", None) + if current is not None: + current = [item[0] for item in current] + loaded.update(current) + return list(loaded) + + +def loaded_lora_str(): + return ", ".join(loaded_lora()) + + +def make_meta(fn, maxrank, rank_ratio): + meta = { + "model_spec.sai_model_spec": "1.0.0", + "model_spec.title": os.path.splitext(os.path.basename(fn))[0], + "model_spec.author": "SD.Next", + "model_spec.implementation": "https://github.com/vladmandic/automatic", + "model_spec.date": datetime.datetime.now().astimezone().replace(microsecond=0).isoformat(), + "model_spec.base_model": shared.opts.sd_model_checkpoint, + "model_spec.dtype": str(devices.dtype), + "model_spec.base_lora": json.dumps(loaded_lora()), + "model_spec.config": f"maxrank={maxrank} rank_ratio={rank_ratio}", + } + if shared.sd_model_type == "sdxl": + meta["model_spec.architecture"] = "stable-diffusion-xl-v1-base/lora" # sai standard + meta["ss_base_model_version"] = "sdxl_base_v1-0" # kohya standard + elif shared.sd_model_type == "sd": + meta["model_spec.architecture"] = "stable-diffusion-v1/lora" + meta["ss_base_model_version"] = "sd_v1" + elif shared.sd_model_type == "f1": + meta["model_spec.architecture"] = "flux-1-dev/lora" + meta["ss_base_model_version"] = "flux1" + elif shared.sd_model_type == "sc": + meta["model_spec.architecture"] = "stable-cascade-v1-prior/lora" + return meta + + +def make_lora(fn, maxrank, auto_rank, rank_ratio, modules, overwrite): + if not shared.sd_loaded or not shared.native: + msg = "LoRA extract: model not loaded" + shared.log.warning(msg) + yield msg + return + if loaded_lora() == "": + msg = "LoRA extract: no LoRA detected" + shared.log.warning(msg) + yield msg + return + if not fn: + msg = "LoRA extract: target filename required" + shared.log.warning(msg) + yield msg + return + t0 = time.time() + maxrank = int(maxrank) + rank_ratio = 1 if not auto_rank else rank_ratio + shared.log.debug(f'LoRA extract: modules={modules} maxrank={maxrank} auto={auto_rank} ratio={rank_ratio} fn="{fn}"') + shared.state.begin('LoRA extract') + + with p.Progress(p.TextColumn('[cyan]LoRA extract'), p.BarColumn(), p.TaskProgressColumn(), p.TimeRemainingColumn(), p.TimeElapsedColumn(), p.TextColumn('[cyan]{task.description}'), console=shared.console) as progress: + + if 'te' in modules and getattr(shared.sd_model, 'text_encoder', None) is not None: + modules = shared.sd_model.text_encoder.named_modules() + task = progress.add_task(description="te1 decompose", total=len(list(modules))) + for name, module in shared.sd_model.text_encoder.named_modules(): + progress.update(task, advance=1) + weights_backup = getattr(module, "network_weights_backup", None) + if weights_backup is None or getattr(module, "network_current_names", None) is None: + continue + prefix = "lora_te1_" if hasattr(shared.sd_model, 'text_encoder_2') else "lora_te_" + module.svdhandler = SVDHandler(maxrank, rank_ratio) + module.svdhandler.network_name = prefix + name.replace(".", "_") + with devices.inference_context(): + module.svdhandler.decompose(module.weight, weights_backup) + progress.remove_task(task) + t1 = time.time() + + if 'te' in modules and getattr(shared.sd_model, 'text_encoder_2', None) is not None: + modules = shared.sd_model.text_encoder_2.named_modules() + task = progress.add_task(description="te2 decompose", total=len(list(modules))) + for name, module in shared.sd_model.text_encoder_2.named_modules(): + progress.update(task, advance=1) + weights_backup = getattr(module, "network_weights_backup", None) + if weights_backup is None or getattr(module, "network_current_names", None) is None: + continue + module.svdhandler = SVDHandler(maxrank, rank_ratio) + module.svdhandler.network_name = "lora_te2_" + name.replace(".", "_") + with devices.inference_context(): + module.svdhandler.decompose(module.weight, weights_backup) + progress.remove_task(task) + t2 = time.time() + + if 'unet' in modules and getattr(shared.sd_model, 'unet', None) is not None: + modules = shared.sd_model.unet.named_modules() + task = progress.add_task(description="unet decompose", total=len(list(modules))) + for name, module in shared.sd_model.unet.named_modules(): + progress.update(task, advance=1) + weights_backup = getattr(module, "network_weights_backup", None) + if weights_backup is None or getattr(module, "network_current_names", None) is None: + continue + module.svdhandler = SVDHandler(maxrank, rank_ratio) + module.svdhandler.network_name = "lora_unet_" + name.replace(".", "_") + with devices.inference_context(): + module.svdhandler.decompose(module.weight, weights_backup) + progress.remove_task(task) + t3 = time.time() + + # TODO: Handle quant for Flux + # if 'te' in modules and getattr(shared.sd_model, 'transformer', None) is not None: + # for name, module in shared.sd_model.transformer.named_modules(): + # if "norm" in name and "linear" not in name: + # continue + # weights_backup = getattr(module, "network_weights_backup", None) + # if weights_backup is None: + # continue + # module.svdhandler = SVDHandler() + # module.svdhandler.network_name = "lora_transformer_" + name.replace(".", "_") + # module.svdhandler.decompose(module.weight, weights_backup) + # module.svdhandler.findrank(rank, rank_ratio) + + lora_state_dict = {} + for sub in ['text_encoder', 'text_encoder_2', 'unet', 'transformer']: + submodel = getattr(shared.sd_model, sub, None) + if submodel is not None: + modules = submodel.named_modules() + task = progress.add_task(description=f"{sub} exctract", total=len(list(modules))) + for _name, module in submodel.named_modules(): + progress.update(task, advance=1) + if not hasattr(module, "svdhandler"): + continue + lora_state_dict.update(module.svdhandler.makeweights()) + del module.svdhandler + progress.remove_task(task) + t4 = time.time() + + if not os.path.isabs(fn): + fn = os.path.join(shared.cmd_opts.lora_dir, fn) + if not fn.endswith('.safetensors'): + fn += '.safetensors' + if os.path.exists(fn): + if overwrite: + os.remove(fn) + else: + msg = f'LoRA extract: fn="{fn}" file exists' + shared.log.warning(msg) + yield msg + return + + shared.state.end() + meta = make_meta(fn, maxrank, rank_ratio) + shared.log.debug(f'LoRA metadata: {meta}') + try: + save_file(tensors=lora_state_dict, metadata=meta, filename=fn) + except Exception as e: + msg = f'LoRA extract error: fn="{fn}" {e}' + shared.log.error(msg) + yield msg + return + t5 = time.time() + shared.log.debug(f'LoRA extract: time={t5-t0:.2f} te1={t1-t0:.2f} te2={t2-t1:.2f} unet={t3-t2:.2f} save={t5-t4:.2f}') + keys = list(lora_state_dict.keys()) + msg = f'LoRA extract: fn="{fn}" keys={len(keys)}' + shared.log.info(msg) + yield msg + + +def create_ui(): + def gr_show(visible=True): + return {"visible": visible, "__type__": "update"} + + with gr.Tab(label="Extract LoRA"): + with gr.Row(): + loaded = gr.Textbox(placeholder="Press refresh to query loaded LoRA", label="Loaded LoRA", interactive=False) + create_refresh_button(loaded, lambda: None, lambda: {'value': loaded_lora_str()}, "testid") + with gr.Group(): + with gr.Row(): + modules = gr.CheckboxGroup(label="Modules to extract", value=['unet'], choices=['te', 'unet']) + with gr.Row(): + auto_rank = gr.Checkbox(value=False, label="Automatically determine rank") + rank_ratio = gr.Slider(label="Autorank ratio", value=1, minimum=0, maximum=1, step=0.05, visible=False) + rank = gr.Slider(label="Maximum rank", value=32, minimum=1, maximum=256) + with gr.Row(): + filename = gr.Textbox(label="LoRA target filename") + overwrite = gr.Checkbox(value=False, label="Overwrite existing file") + with gr.Row(): + extract = gr.Button(value="Extract LoRA", variant='primary') + status = gr.HTML(value="", show_label=False) + + auto_rank.change(fn=lambda x: gr_show(x), inputs=[auto_rank], outputs=[rank_ratio]) + extract.click( + fn=wrap_gradio_gpu_call(make_lora, extra_outputs=[]), + inputs=[filename, rank, auto_rank, rank_ratio, modules, overwrite], + outputs=[status] + ) diff --git a/modules/lora/lyco_helpers.py b/modules/lora/lyco_helpers.py new file mode 100644 index 000000000..9a16d25ab --- /dev/null +++ b/modules/lora/lyco_helpers.py @@ -0,0 +1,66 @@ +import torch + + +def make_weight_cp(t, wa, wb): + temp = torch.einsum('i j k l, j r -> i r k l', t, wb) + return torch.einsum('i j k l, i r -> r j k l', temp, wa) + + +def rebuild_conventional(up, down, shape, dyn_dim=None): + up = up.reshape(up.size(0), -1) + down = down.reshape(down.size(0), -1) + if dyn_dim is not None: + up = up[:, :dyn_dim] + down = down[:dyn_dim, :] + return (up @ down).reshape(shape) + + +def rebuild_cp_decomposition(up, down, mid): + up = up.reshape(up.size(0), -1) + down = down.reshape(down.size(0), -1) + return torch.einsum('n m k l, i n, m j -> i j k l', mid, up, down) + + +# copied from https://github.com/KohakuBlueleaf/LyCORIS/blob/dev/lycoris/modules/lokr.py +def factorization(dimension: int, factor:int=-1) -> tuple[int, int]: + """ + return a tuple of two value of input dimension decomposed by the number closest to factor + second value is higher or equal than first value. + + In LoRA with Kroneckor Product, first value is a value for weight scale. + secon value is a value for weight. + + Becuase of non-commutative property, A⊗B ≠ B⊗A. Meaning of two matrices is slightly different. + + examples + factor + -1 2 4 8 16 ... + 127 -> 1, 127 127 -> 1, 127 127 -> 1, 127 127 -> 1, 127 127 -> 1, 127 + 128 -> 8, 16 128 -> 2, 64 128 -> 4, 32 128 -> 8, 16 128 -> 8, 16 + 250 -> 10, 25 250 -> 2, 125 250 -> 2, 125 250 -> 5, 50 250 -> 10, 25 + 360 -> 8, 45 360 -> 2, 180 360 -> 4, 90 360 -> 8, 45 360 -> 12, 30 + 512 -> 16, 32 512 -> 2, 256 512 -> 4, 128 512 -> 8, 64 512 -> 16, 32 + 1024 -> 32, 32 1024 -> 2, 512 1024 -> 4, 256 1024 -> 8, 128 1024 -> 16, 64 + """ + + if factor > 0 and (dimension % factor) == 0: + m = factor + n = dimension // factor + if m > n: + n, m = m, n + return m, n + if factor < 0: + factor = dimension + m, n = 1, dimension + length = m + n + while m length or new_m>factor: + break + m, n = new_m, new_n + if m > n: + n, m = m, n + return m, n diff --git a/modules/lora/network.py b/modules/lora/network.py new file mode 100644 index 000000000..0785ef9f4 --- /dev/null +++ b/modules/lora/network.py @@ -0,0 +1,187 @@ +import os +from collections import namedtuple +import enum + +from modules import sd_models, hashes, shared + +NetworkWeights = namedtuple('NetworkWeights', ['network_key', 'sd_key', 'w', 'sd_module']) +metadata_tags_order = {"ss_sd_model_name": 1, "ss_resolution": 2, "ss_clip_skip": 3, "ss_num_train_images": 10, "ss_tag_frequency": 20} + + +class SdVersion(enum.Enum): + Unknown = 1 + SD1 = 2 + SD2 = 3 + SD3 = 3 + SDXL = 4 + SC = 5 + F1 = 6 + + +class NetworkOnDisk: + def __init__(self, name, filename): + self.shorthash = None + self.hash = None + self.name = name + self.filename = filename + if filename.startswith(shared.cmd_opts.lora_dir): + self.fullname = os.path.splitext(filename[len(shared.cmd_opts.lora_dir):].strip("/"))[0] + else: + self.fullname = name + self.metadata = {} + self.is_safetensors = os.path.splitext(filename)[1].lower() == ".safetensors" + if self.is_safetensors: + self.metadata = sd_models.read_metadata_from_safetensors(filename) + if self.metadata: + m = {} + for k, v in sorted(self.metadata.items(), key=lambda x: metadata_tags_order.get(x[0], 999)): + m[k] = v + self.metadata = m + self.alias = self.metadata.get('ss_output_name', self.name) + sha256 = hashes.sha256_from_cache(self.filename, "lora/" + self.name) or hashes.sha256_from_cache(self.filename, "lora/" + self.name, use_addnet_hash=True) or self.metadata.get('sshs_model_hash') + self.set_hash(sha256) + self.sd_version = self.detect_version() + + def detect_version(self): + base = str(self.metadata.get('ss_base_model_version', "")).lower() + arch = str(self.metadata.get('modelspec.architecture', "")).lower() + if base.startswith("sd_v1"): + return 'sd1' + if base.startswith("sdxl"): + return 'xl' + if base.startswith("stable_cascade"): + return 'sc' + if base.startswith("sd3"): + return 'sd3' + if base.startswith("flux"): + return 'f1' + + if arch.startswith("stable-diffusion-v1"): + return 'sd1' + if arch.startswith("stable-diffusion-xl"): + return 'xl' + if arch.startswith("stable-cascade"): + return 'sc' + if arch.startswith("flux"): + return 'f1' + + if "v1-5" in str(self.metadata.get('ss_sd_model_name', "")): + return 'sd1' + if str(self.metadata.get('ss_v2', "")) == "True": + return 'sd2' + if 'flux' in self.name.lower(): + return 'f1' + if 'xl' in self.name.lower(): + return 'xl' + + return '' + + def set_hash(self, v): + self.hash = v or '' + self.shorthash = self.hash[0:8] + + def read_hash(self): + if not self.hash: + self.set_hash(hashes.sha256(self.filename, "lora/" + self.name, use_addnet_hash=self.is_safetensors) or '') + + def get_alias(self): + import modules.lora.networks as networks + return self.name if shared.opts.lora_preferred_name == "filename" or self.alias.lower() in networks.forbidden_network_aliases else self.alias + + +class Network: # LoraModule + def __init__(self, name, network_on_disk: NetworkOnDisk): + self.name = name + self.network_on_disk = network_on_disk + self.te_multiplier = 1.0 + self.unet_multiplier = [1.0] * 3 + self.dyn_dim = None + self.modules = {} + self.bundle_embeddings = {} + self.mtime = None + self.mentioned_name = None + self.tags = None + """the text that was used to add the network to prompt - can be either name or an alias""" + + +class ModuleType: + def create_module(self, net: Network, weights: NetworkWeights) -> Network | None: # pylint: disable=W0613 + return None + + +class NetworkModule: + def __init__(self, net: Network, weights: NetworkWeights): + self.network = net + self.network_key = weights.network_key + self.sd_key = weights.sd_key + self.sd_module = weights.sd_module + if hasattr(self.sd_module, 'weight'): + self.shape = self.sd_module.weight.shape + self.dim = None + self.bias = weights.w.get("bias") + self.alpha = weights.w["alpha"].item() if "alpha" in weights.w else None + self.scale = weights.w["scale"].item() if "scale" in weights.w else None + self.dora_scale = weights.w.get("dora_scale", None) + self.dora_norm_dims = len(self.shape) - 1 + + def multiplier(self): + unet_multiplier = 3 * [self.network.unet_multiplier] if not isinstance(self.network.unet_multiplier, list) else self.network.unet_multiplier + if 'transformer' in self.sd_key[:20]: + return self.network.te_multiplier + if "down_blocks" in self.sd_key: + return unet_multiplier[0] + if "mid_block" in self.sd_key: + return unet_multiplier[1] + if "up_blocks" in self.sd_key: + return unet_multiplier[2] + else: + return unet_multiplier[0] + + def calc_scale(self): + if self.scale is not None: + return self.scale + if self.dim is not None and self.alpha is not None: + return self.alpha / self.dim + return 1.0 + + def apply_weight_decompose(self, updown, orig_weight): + # Match the device/dtype + orig_weight = orig_weight.to(updown.dtype) + dora_scale = self.dora_scale.to(device=orig_weight.device, dtype=updown.dtype) + updown = updown.to(orig_weight.device) + + merged_scale1 = updown + orig_weight + merged_scale1_norm = ( + merged_scale1.transpose(0, 1) + .reshape(merged_scale1.shape[1], -1) + .norm(dim=1, keepdim=True) + .reshape(merged_scale1.shape[1], *[1] * self.dora_norm_dims) + .transpose(0, 1) + ) + + dora_merged = ( + merged_scale1 * (dora_scale / merged_scale1_norm) + ) + final_updown = dora_merged - orig_weight + return final_updown + + def finalize_updown(self, updown, orig_weight, output_shape, ex_bias=None): + if self.bias is not None: + updown = updown.reshape(self.bias.shape) + updown += self.bias.to(orig_weight.device, dtype=orig_weight.dtype) + updown = updown.reshape(output_shape) + if len(output_shape) == 4: + updown = updown.reshape(output_shape) + if orig_weight.size().numel() == updown.size().numel(): + updown = updown.reshape(orig_weight.shape) + if ex_bias is not None: + ex_bias = ex_bias * self.multiplier() + if self.dora_scale is not None: + updown = self.apply_weight_decompose(updown, orig_weight) + return updown * self.calc_scale() * self.multiplier(), ex_bias + + def calc_updown(self, target): + raise NotImplementedError + + def forward(self, x, y): + raise NotImplementedError diff --git a/modules/lora/network_full.py b/modules/lora/network_full.py new file mode 100644 index 000000000..5eb0b2e4e --- /dev/null +++ b/modules/lora/network_full.py @@ -0,0 +1,26 @@ +import modules.lora.network as network + + +class ModuleTypeFull(network.ModuleType): + def create_module(self, net: network.Network, weights: network.NetworkWeights): + if all(x in weights.w for x in ["diff"]): + return NetworkModuleFull(net, weights) + return None + + +class NetworkModuleFull(network.NetworkModule): # pylint: disable=abstract-method + def __init__(self, net: network.Network, weights: network.NetworkWeights): + super().__init__(net, weights) + + self.weight = weights.w.get("diff") + self.ex_bias = weights.w.get("diff_b") + + def calc_updown(self, target): + output_shape = self.weight.shape + updown = self.weight.to(target.device, dtype=target.dtype) + if self.ex_bias is not None: + ex_bias = self.ex_bias.to(target.device, dtype=target.dtype) + else: + ex_bias = None + + return self.finalize_updown(updown, target, output_shape, ex_bias) diff --git a/modules/lora/network_glora.py b/modules/lora/network_glora.py new file mode 100644 index 000000000..ffcb25986 --- /dev/null +++ b/modules/lora/network_glora.py @@ -0,0 +1,30 @@ +import modules.lora.network as network + + +class ModuleTypeGLora(network.ModuleType): + def create_module(self, net: network.Network, weights: network.NetworkWeights): + if all(x in weights.w for x in ["a1.weight", "a2.weight", "alpha", "b1.weight", "b2.weight"]): + return NetworkModuleGLora(net, weights) + return None + +# adapted from https://github.com/KohakuBlueleaf/LyCORIS +class NetworkModuleGLora(network.NetworkModule): # pylint: disable=abstract-method + def __init__(self, net: network.Network, weights: network.NetworkWeights): + super().__init__(net, weights) + + if hasattr(self.sd_module, 'weight'): + self.shape = self.sd_module.weight.shape + + self.w1a = weights.w["a1.weight"] + self.w1b = weights.w["b1.weight"] + self.w2a = weights.w["a2.weight"] + self.w2b = weights.w["b2.weight"] + + def calc_updown(self, target): # pylint: disable=arguments-differ + w1a = self.w1a.to(target.device, dtype=target.dtype) + w1b = self.w1b.to(target.device, dtype=target.dtype) + w2a = self.w2a.to(target.device, dtype=target.dtype) + w2b = self.w2b.to(target.device, dtype=target.dtype) + output_shape = [w1a.size(0), w1b.size(1)] + updown = (w2b @ w1b) + ((target @ w2a) @ w1a) + return self.finalize_updown(updown, target, output_shape) diff --git a/modules/lora/network_hada.py b/modules/lora/network_hada.py new file mode 100644 index 000000000..6fc142b3b --- /dev/null +++ b/modules/lora/network_hada.py @@ -0,0 +1,46 @@ +import modules.lora.lyco_helpers as lyco_helpers +import modules.lora.network as network + + +class ModuleTypeHada(network.ModuleType): + def create_module(self, net: network.Network, weights: network.NetworkWeights): + if all(x in weights.w for x in ["hada_w1_a", "hada_w1_b", "hada_w2_a", "hada_w2_b"]): + return NetworkModuleHada(net, weights) + return None + + +class NetworkModuleHada(network.NetworkModule): # pylint: disable=abstract-method + def __init__(self, net: network.Network, weights: network.NetworkWeights): + super().__init__(net, weights) + if hasattr(self.sd_module, 'weight'): + self.shape = self.sd_module.weight.shape + self.w1a = weights.w["hada_w1_a"] + self.w1b = weights.w["hada_w1_b"] + self.dim = self.w1b.shape[0] + self.w2a = weights.w["hada_w2_a"] + self.w2b = weights.w["hada_w2_b"] + self.t1 = weights.w.get("hada_t1") + self.t2 = weights.w.get("hada_t2") + + def calc_updown(self, target): + w1a = self.w1a.to(target.device, dtype=target.dtype) + w1b = self.w1b.to(target.device, dtype=target.dtype) + w2a = self.w2a.to(target.device, dtype=target.dtype) + w2b = self.w2b.to(target.device, dtype=target.dtype) + output_shape = [w1a.size(0), w1b.size(1)] + if self.t1 is not None: + output_shape = [w1a.size(1), w1b.size(1)] + t1 = self.t1.to(target.device, dtype=target.dtype) + updown1 = lyco_helpers.make_weight_cp(t1, w1a, w1b) + output_shape += t1.shape[2:] + else: + if len(w1b.shape) == 4: + output_shape += w1b.shape[2:] + updown1 = lyco_helpers.rebuild_conventional(w1a, w1b, output_shape) + if self.t2 is not None: + t2 = self.t2.to(target.device, dtype=target.dtype) + updown2 = lyco_helpers.make_weight_cp(t2, w2a, w2b) + else: + updown2 = lyco_helpers.rebuild_conventional(w2a, w2b, output_shape) + updown = updown1 * updown2 + return self.finalize_updown(updown, target, output_shape) diff --git a/modules/lora/network_ia3.py b/modules/lora/network_ia3.py new file mode 100644 index 000000000..479e42526 --- /dev/null +++ b/modules/lora/network_ia3.py @@ -0,0 +1,24 @@ +import modules.lora.network as network + +class ModuleTypeIa3(network.ModuleType): + def create_module(self, net: network.Network, weights: network.NetworkWeights): + if all(x in weights.w for x in ["weight"]): + return NetworkModuleIa3(net, weights) + return None + + +class NetworkModuleIa3(network.NetworkModule): # pylint: disable=abstract-method + def __init__(self, net: network.Network, weights: network.NetworkWeights): + super().__init__(net, weights) + self.w = weights.w["weight"] + self.on_input = weights.w["on_input"].item() + + def calc_updown(self, target): + w = self.w.to(target.device, dtype=target.dtype) + output_shape = [w.size(0), target.size(1)] + if self.on_input: + output_shape.reverse() + else: + w = w.reshape(-1, 1) + updown = target * w + return self.finalize_updown(updown, target, output_shape) diff --git a/modules/lora/network_lokr.py b/modules/lora/network_lokr.py new file mode 100644 index 000000000..877d4005b --- /dev/null +++ b/modules/lora/network_lokr.py @@ -0,0 +1,57 @@ +import torch +import modules.lora.lyco_helpers as lyco_helpers +import modules.lora.network as network + + +class ModuleTypeLokr(network.ModuleType): + def create_module(self, net: network.Network, weights: network.NetworkWeights): + has_1 = "lokr_w1" in weights.w or ("lokr_w1_a" in weights.w and "lokr_w1_b" in weights.w) + has_2 = "lokr_w2" in weights.w or ("lokr_w2_a" in weights.w and "lokr_w2_b" in weights.w) + if has_1 and has_2: + return NetworkModuleLokr(net, weights) + return None + + +def make_kron(orig_shape, w1, w2): + if len(w2.shape) == 4: + w1 = w1.unsqueeze(2).unsqueeze(2) + w2 = w2.contiguous() + return torch.kron(w1, w2).reshape(orig_shape) + + +class NetworkModuleLokr(network.NetworkModule): # pylint: disable=abstract-method + def __init__(self, net: network.Network, weights: network.NetworkWeights): + super().__init__(net, weights) + self.w1 = weights.w.get("lokr_w1") + self.w1a = weights.w.get("lokr_w1_a") + self.w1b = weights.w.get("lokr_w1_b") + self.dim = self.w1b.shape[0] if self.w1b is not None else self.dim + self.w2 = weights.w.get("lokr_w2") + self.w2a = weights.w.get("lokr_w2_a") + self.w2b = weights.w.get("lokr_w2_b") + self.dim = self.w2b.shape[0] if self.w2b is not None else self.dim + self.t2 = weights.w.get("lokr_t2") + + def calc_updown(self, target): + if self.w1 is not None: + w1 = self.w1.to(target.device, dtype=target.dtype) + else: + w1a = self.w1a.to(target.device, dtype=target.dtype) + w1b = self.w1b.to(target.device, dtype=target.dtype) + w1 = w1a @ w1b + if self.w2 is not None: + w2 = self.w2.to(target.device, dtype=target.dtype) + elif self.t2 is None: + w2a = self.w2a.to(target.device, dtype=target.dtype) + w2b = self.w2b.to(target.device, dtype=target.dtype) + w2 = w2a @ w2b + else: + t2 = self.t2.to(target.device, dtype=target.dtype) + w2a = self.w2a.to(target.device, dtype=target.dtype) + w2b = self.w2b.to(target.device, dtype=target.dtype) + w2 = lyco_helpers.make_weight_cp(t2, w2a, w2b) + output_shape = [w1.size(0) * w2.size(0), w1.size(1) * w2.size(1)] + if len(target.shape) == 4: + output_shape = target.shape + updown = make_kron(output_shape, w1, w2) + return self.finalize_updown(updown, target, output_shape) diff --git a/modules/lora/network_lora.py b/modules/lora/network_lora.py new file mode 100644 index 000000000..6c1d7ea3f --- /dev/null +++ b/modules/lora/network_lora.py @@ -0,0 +1,78 @@ +import torch +import diffusers.models.lora as diffusers_lora +import modules.lora.lyco_helpers as lyco_helpers +import modules.lora.network as network +from modules import devices + + +class ModuleTypeLora(network.ModuleType): + def create_module(self, net: network.Network, weights: network.NetworkWeights): + if all(x in weights.w for x in ["lora_up.weight", "lora_down.weight"]): + return NetworkModuleLora(net, weights) + return None + + +class NetworkModuleLora(network.NetworkModule): + + def __init__(self, net: network.Network, weights: network.NetworkWeights): + super().__init__(net, weights) + self.up_model = self.create_module(weights.w, "lora_up.weight") + self.down_model = self.create_module(weights.w, "lora_down.weight") + self.mid_model = self.create_module(weights.w, "lora_mid.weight", none_ok=True) + self.dim = weights.w["lora_down.weight"].shape[0] + + def create_module(self, weights, key, none_ok=False): + from modules.shared import opts + weight = weights.get(key) + if weight is None and none_ok: + return None + linear_modules = [torch.nn.Linear, torch.nn.modules.linear.NonDynamicallyQuantizableLinear, torch.nn.MultiheadAttention, diffusers_lora.LoRACompatibleLinear] + is_linear = type(self.sd_module) in linear_modules or self.sd_module.__class__.__name__ in {"NNCFLinear", "QLinear", "Linear4bit"} + is_conv = type(self.sd_module) in [torch.nn.Conv2d, diffusers_lora.LoRACompatibleConv] or self.sd_module.__class__.__name__ in {"NNCFConv2d", "QConv2d"} + if is_linear: + weight = weight.reshape(weight.shape[0], -1) + module = torch.nn.Linear(weight.shape[1], weight.shape[0], bias=False) + elif is_conv and key == "lora_down.weight" or key == "dyn_up": + if len(weight.shape) == 2: + weight = weight.reshape(weight.shape[0], -1, 1, 1) + if weight.shape[2] != 1 or weight.shape[3] != 1: + module = torch.nn.Conv2d(weight.shape[1], weight.shape[0], self.sd_module.kernel_size, self.sd_module.stride, self.sd_module.padding, bias=False) + else: + module = torch.nn.Conv2d(weight.shape[1], weight.shape[0], (1, 1), bias=False) + elif is_conv and key == "lora_mid.weight": + module = torch.nn.Conv2d(weight.shape[1], weight.shape[0], self.sd_module.kernel_size, self.sd_module.stride, self.sd_module.padding, bias=False) + elif is_conv and key == "lora_up.weight" or key == "dyn_down": + module = torch.nn.Conv2d(weight.shape[1], weight.shape[0], (1, 1), bias=False) + else: + raise AssertionError(f'Lora unsupported: layer={self.network_key} type={type(self.sd_module).__name__}') + with torch.no_grad(): + if weight.shape != module.weight.shape: + weight = weight.reshape(module.weight.shape) + module.weight.copy_(weight) + if opts.lora_load_gpu: + module = module.to(device=devices.device, dtype=devices.dtype) + module.weight.requires_grad_(False) + return module + + def calc_updown(self, target): # pylint: disable=W0237 + target_dtype = target.dtype if target.dtype != torch.uint8 else self.up_model.weight.dtype + up = self.up_model.weight.to(target.device, dtype=target_dtype) + down = self.down_model.weight.to(target.device, dtype=target_dtype) + output_shape = [up.size(0), down.size(1)] + if self.mid_model is not None: + # cp-decomposition + mid = self.mid_model.weight.to(target.device, dtype=target_dtype) + updown = lyco_helpers.rebuild_cp_decomposition(up, down, mid) + output_shape += mid.shape[2:] + else: + if len(down.shape) == 4: + output_shape += down.shape[2:] + updown = lyco_helpers.rebuild_conventional(up, down, output_shape, self.network.dyn_dim) + return self.finalize_updown(updown, target, output_shape) + + def forward(self, x, y): + self.up_model.to(device=devices.device) + self.down_model.to(device=devices.device) + if hasattr(y, "scale"): + return y(scale=1) + self.up_model(self.down_model(x)) * self.multiplier() * self.calc_scale() + return y + self.up_model(self.down_model(x)) * self.multiplier() * self.calc_scale() diff --git a/modules/lora/network_norm.py b/modules/lora/network_norm.py new file mode 100644 index 000000000..e8f1740e3 --- /dev/null +++ b/modules/lora/network_norm.py @@ -0,0 +1,23 @@ +import modules.lora.network as network + +class ModuleTypeNorm(network.ModuleType): + def create_module(self, net: network.Network, weights: network.NetworkWeights): + if all(x in weights.w for x in ["w_norm", "b_norm"]): + return NetworkModuleNorm(net, weights) + return None + + +class NetworkModuleNorm(network.NetworkModule): # pylint: disable=abstract-method + def __init__(self, net: network.Network, weights: network.NetworkWeights): + super().__init__(net, weights) + self.w_norm = weights.w.get("w_norm") + self.b_norm = weights.w.get("b_norm") + + def calc_updown(self, target): + output_shape = self.w_norm.shape + updown = self.w_norm.to(target.device, dtype=target.dtype) + if self.b_norm is not None: + ex_bias = self.b_norm.to(target.device, dtype=target.dtype) + else: + ex_bias = None + return self.finalize_updown(updown, target, output_shape, ex_bias) diff --git a/modules/lora/network_oft.py b/modules/lora/network_oft.py new file mode 100644 index 000000000..808286066 --- /dev/null +++ b/modules/lora/network_oft.py @@ -0,0 +1,81 @@ +import torch +import modules.lora.network as network +from modules.lora.lyco_helpers import factorization +from einops import rearrange + + +class ModuleTypeOFT(network.ModuleType): + def create_module(self, net: network.Network, weights: network.NetworkWeights): + if all(x in weights.w for x in ["oft_blocks"]) or all(x in weights.w for x in ["oft_diag"]): + return NetworkModuleOFT(net, weights) + return None + +# Supports both kohya-ss' implementation of COFT https://github.com/kohya-ss/sd-scripts/blob/main/networks/oft.py +# and KohakuBlueleaf's implementation of OFT/COFT https://github.com/KohakuBlueleaf/LyCORIS/blob/dev/lycoris/modules/diag_oft.py +class NetworkModuleOFT(network.NetworkModule): # pylint: disable=abstract-method + def __init__(self, net: network.Network, weights: network.NetworkWeights): + super().__init__(net, weights) + self.lin_module = None + self.org_module: list[torch.Module] = [self.sd_module] + self.scale = 1.0 + + # kohya-ss + if "oft_blocks" in weights.w.keys(): + self.is_kohya = True + self.oft_blocks = weights.w["oft_blocks"] # (num_blocks, block_size, block_size) + self.alpha = weights.w["alpha"] # alpha is constraint + self.dim = self.oft_blocks.shape[0] # lora dim + # LyCORIS + elif "oft_diag" in weights.w.keys(): + self.is_kohya = False + self.oft_blocks = weights.w["oft_diag"] + # self.alpha is unused + self.dim = self.oft_blocks.shape[1] # (num_blocks, block_size, block_size) + + is_linear = type(self.sd_module) in [torch.nn.Linear, torch.nn.modules.linear.NonDynamicallyQuantizableLinear] + is_conv = type(self.sd_module) in [torch.nn.Conv2d] + is_other_linear = type(self.sd_module) in [torch.nn.MultiheadAttention] # unsupported + + if is_linear: + self.out_dim = self.sd_module.out_features + elif is_conv: + self.out_dim = self.sd_module.out_channels + elif is_other_linear: + self.out_dim = self.sd_module.embed_dim + + if self.is_kohya: + self.constraint = self.alpha * self.out_dim + self.num_blocks = self.dim + self.block_size = self.out_dim // self.dim + else: + self.constraint = None + self.block_size, self.num_blocks = factorization(self.out_dim, self.dim) + + def calc_updown(self, target): + oft_blocks = self.oft_blocks.to(target.device, dtype=target.dtype) + eye = torch.eye(self.block_size, device=target.device) + constraint = self.constraint.to(target.device) + + if self.is_kohya: + block_Q = oft_blocks - oft_blocks.transpose(1, 2) # ensure skew-symmetric orthogonal matrix + norm_Q = torch.norm(block_Q.flatten()).to(target.device) + new_norm_Q = torch.clamp(norm_Q, max=constraint) + block_Q = block_Q * ((new_norm_Q + 1e-8) / (norm_Q + 1e-8)) + mat1 = eye + block_Q + mat2 = (eye - block_Q).float().inverse() + oft_blocks = torch.matmul(mat1, mat2) + + R = oft_blocks.to(target.device, dtype=target.dtype) + + # This errors out for MultiheadAttention, might need to be handled up-stream + merged_weight = rearrange(target, '(k n) ... -> k n ...', k=self.num_blocks, n=self.block_size) + merged_weight = torch.einsum( + 'k n m, k n ... -> k m ...', + R, + merged_weight + ) + merged_weight = rearrange(merged_weight, 'k m ... -> (k m) ...') + + updown = merged_weight.to(target.device, dtype=target.dtype) - target + output_shape = target.shape + return self.finalize_updown(updown, target, output_shape) diff --git a/modules/lora/network_overrides.py b/modules/lora/network_overrides.py new file mode 100644 index 000000000..5334f3c1b --- /dev/null +++ b/modules/lora/network_overrides.py @@ -0,0 +1,49 @@ +from modules import shared + + +maybe_diffusers = [ # forced if lora_maybe_diffusers is enabled + 'aaebf6360f7d', # sd15-lcm + '3d18b05e4f56', # sdxl-lcm + 'b71dcb732467', # sdxl-tcd + '813ea5fb1c67', # sdxl-turbo + # not really needed, but just in case + '5a48ac366664', # hyper-sd15-1step + 'ee0ff23dcc42', # hyper-sd15-2step + 'e476eb1da5df', # hyper-sd15-4step + 'ecb844c3f3b0', # hyper-sd15-8step + '1ab289133ebb', # hyper-sd15-8step-cfg + '4f494295edb1', # hyper-sdxl-8step + 'ca14a8c621f8', # hyper-sdxl-8step-cfg + '1c88f7295856', # hyper-sdxl-4step + 'fdd5dcd1d88a', # hyper-sdxl-2step + '8cca3706050b', # hyper-sdxl-1step +] + +force_diffusers = [ # forced always + '816d0eed49fd', # flash-sdxl + 'c2ec22757b46', # flash-sd15 +] + +force_models = [ # forced always + 'sc', + # 'sd3', + 'kandinsky', + 'hunyuandit', + 'auraflow', +] + +force_classes = [ # forced always +] + + +def check_override(shorthash=''): + force = False + force = force or (shared.sd_model_type in force_models) + force = force or (shared.sd_model.__class__.__name__ in force_classes) + if len(shorthash) < 4: + return force + force = force or (any(x.startswith(shorthash) for x in maybe_diffusers) if shared.opts.lora_maybe_diffusers else False) + force = force or any(x.startswith(shorthash) for x in force_diffusers) + if force and shared.opts.lora_maybe_diffusers: + shared.log.debug('LoRA override: force diffusers') + return force diff --git a/modules/lora/networks.py b/modules/lora/networks.py new file mode 100644 index 000000000..762705b67 --- /dev/null +++ b/modules/lora/networks.py @@ -0,0 +1,453 @@ +from typing import Union, List +import os +import re +import time +import concurrent +import modules.lora.network as network +import modules.lora.network_lora as network_lora +import modules.lora.network_hada as network_hada +import modules.lora.network_ia3 as network_ia3 +import modules.lora.network_oft as network_oft +import modules.lora.network_lokr as network_lokr +import modules.lora.network_full as network_full +import modules.lora.network_norm as network_norm +import modules.lora.network_glora as network_glora +import modules.lora.network_overrides as network_overrides +import modules.lora.lora_convert as lora_convert +import torch +import diffusers.models.lora +from modules import shared, devices, sd_models, sd_models_compile, errors, scripts, files_cache, model_quant + + +debug = os.environ.get('SD_LORA_DEBUG', None) is not None +extra_network_lora = None +available_networks = {} +available_network_aliases = {} +loaded_networks: List[network.Network] = [] +timer = { 'load': 0, 'apply': 0, 'restore': 0, 'deactivate': 0 } +lora_cache = {} +diffuser_loaded = [] +diffuser_scales = [] +available_network_hash_lookup = {} +forbidden_network_aliases = {} +re_network_name = re.compile(r"(.*)\s*\([0-9a-fA-F]+\)") +module_types = [ + network_lora.ModuleTypeLora(), + network_hada.ModuleTypeHada(), + network_ia3.ModuleTypeIa3(), + network_oft.ModuleTypeOFT(), + network_lokr.ModuleTypeLokr(), + network_full.ModuleTypeFull(), + network_norm.ModuleTypeNorm(), + network_glora.ModuleTypeGLora(), +] + + +def assign_network_names_to_compvis_modules(sd_model): + if sd_model is None: + return + sd_model = getattr(shared.sd_model, "pipe", shared.sd_model) # wrapped model compatiblility + network_layer_mapping = {} + if hasattr(sd_model, 'text_encoder') and sd_model.text_encoder is not None: + for name, module in sd_model.text_encoder.named_modules(): + prefix = "lora_te1_" if hasattr(sd_model, 'text_encoder_2') else "lora_te_" + network_name = prefix + name.replace(".", "_") + network_layer_mapping[network_name] = module + module.network_layer_name = network_name + if hasattr(sd_model, 'text_encoder_2'): + for name, module in sd_model.text_encoder_2.named_modules(): + network_name = "lora_te2_" + name.replace(".", "_") + network_layer_mapping[network_name] = module + module.network_layer_name = network_name + if hasattr(sd_model, 'unet'): + for name, module in sd_model.unet.named_modules(): + network_name = "lora_unet_" + name.replace(".", "_") + network_layer_mapping[network_name] = module + module.network_layer_name = network_name + if hasattr(sd_model, 'transformer'): + for name, module in sd_model.transformer.named_modules(): + network_name = "lora_transformer_" + name.replace(".", "_") + network_layer_mapping[network_name] = module + if "norm" in network_name and "linear" not in network_name and shared.sd_model_type != "sd3": + continue + module.network_layer_name = network_name + shared.sd_model.network_layer_mapping = network_layer_mapping + + +def load_diffusers(name, network_on_disk, lora_scale=shared.opts.extra_networks_default_multiplier) -> network.Network | None: + name = name.replace(".", "_") + shared.log.debug(f'Load network: type=LoRA name="{name}" file="{network_on_disk.filename}" detected={network_on_disk.sd_version} method=diffusers scale={lora_scale} fuse={shared.opts.lora_fuse_diffusers}') + if not shared.native: + return None + if not hasattr(shared.sd_model, 'load_lora_weights'): + shared.log.error(f'Load network: type=LoRA class={shared.sd_model.__class__} does not implement load lora') + return None + try: + shared.sd_model.load_lora_weights(network_on_disk.filename, adapter_name=name) + except Exception as e: + if 'already in use' in str(e): + pass + else: + if 'The following keys have not been correctly renamed' in str(e): + shared.log.error(f'Load network: type=LoRA name="{name}" diffusers unsupported format') + else: + shared.log.error(f'Load network: type=LoRA name="{name}" {e}') + if debug: + errors.display(e, "LoRA") + return None + if name not in diffuser_loaded: + diffuser_loaded.append(name) + diffuser_scales.append(lora_scale) + net = network.Network(name, network_on_disk) + net.mtime = os.path.getmtime(network_on_disk.filename) + return net + + +def load_network(name, network_on_disk) -> network.Network | None: + if not shared.sd_loaded: + return None + + cached = lora_cache.get(name, None) + if debug: + shared.log.debug(f'Load network: type=LoRA name="{name}" file="{network_on_disk.filename}" type=lora {"cached" if cached else ""}') + if cached is not None: + return cached + net = network.Network(name, network_on_disk) + net.mtime = os.path.getmtime(network_on_disk.filename) + sd = sd_models.read_state_dict(network_on_disk.filename, what='network') + if shared.sd_model_type == 'f1': # if kohya flux lora, convert state_dict + sd = lora_convert._convert_kohya_flux_lora_to_diffusers(sd) or sd # pylint: disable=protected-access + if shared.sd_model_type == 'sd3': # if kohya flux lora, convert state_dict + try: + sd = lora_convert._convert_kohya_sd3_lora_to_diffusers(sd) or sd # pylint: disable=protected-access + except ValueError: # EAFP for diffusers PEFT keys + pass + assign_network_names_to_compvis_modules(shared.sd_model) + keys_failed_to_match = {} + matched_networks = {} + bundle_embeddings = {} + convert = lora_convert.KeyConvert() + for key_network, weight in sd.items(): + parts = key_network.split('.') + if parts[0] == "bundle_emb": + emb_name, vec_name = parts[1], key_network.split(".", 2)[-1] + emb_dict = bundle_embeddings.get(emb_name, {}) + emb_dict[vec_name] = weight + bundle_embeddings[emb_name] = emb_dict + continue + if len(parts) > 5: # messy handler for diffusers peft lora + key_network_without_network_parts = '_'.join(parts[:-2]) + if not key_network_without_network_parts.startswith('lora_'): + key_network_without_network_parts = 'lora_' + key_network_without_network_parts + network_part = '.'.join(parts[-2:]).replace('lora_A', 'lora_down').replace('lora_B', 'lora_up') + else: + key_network_without_network_parts, network_part = key_network.split(".", 1) + key, sd_module = convert(key_network_without_network_parts) + if sd_module is None: + keys_failed_to_match[key_network] = key + continue + if key not in matched_networks: + matched_networks[key] = network.NetworkWeights(network_key=key_network, sd_key=key, w={}, sd_module=sd_module) + matched_networks[key].w[network_part] = weight + network_types = [] + for key, weights in matched_networks.items(): + net_module = None + for nettype in module_types: + net_module = nettype.create_module(net, weights) + if net_module is not None: + network_types.append(nettype.__class__.__name__) + break + if net_module is None: + shared.log.error(f'LoRA unhandled: name={name} key={key} weights={weights.w.keys()}') + else: + net.modules[key] = net_module + if len(keys_failed_to_match) > 0: + shared.log.warning(f'LoRA name="{name}" type={set(network_types)} unmatched={len(keys_failed_to_match)} matched={len(matched_networks)}') + if debug: + shared.log.debug(f'LoRA name="{name}" unmatched={keys_failed_to_match}') + else: + shared.log.debug(f'LoRA name="{name}" type={set(network_types)} keys={len(matched_networks)}') + if len(matched_networks) == 0: + return None + lora_cache[name] = net + net.bundle_embeddings = bundle_embeddings + return net + +def maybe_recompile_model(names, te_multipliers): + recompile_model = False + if shared.compiled_model_state is not None and shared.compiled_model_state.is_compiled: + if len(names) == len(shared.compiled_model_state.lora_model): + for i, name in enumerate(names): + if shared.compiled_model_state.lora_model[ + i] != f"{name}:{te_multipliers[i] if te_multipliers else shared.opts.extra_networks_default_multiplier}": + recompile_model = True + shared.compiled_model_state.lora_model = [] + break + if not recompile_model: + if len(loaded_networks) > 0 and debug: + shared.log.debug('Model Compile: Skipping LoRa loading') + return + else: + recompile_model = True + shared.compiled_model_state.lora_model = [] + if recompile_model: + backup_cuda_compile = shared.opts.cuda_compile + sd_models.unload_model_weights(op='model') + shared.opts.cuda_compile = [] + sd_models.reload_model_weights(op='model') + shared.opts.cuda_compile = backup_cuda_compile + return recompile_model + + +def load_networks(names, te_multipliers=None, unet_multipliers=None, dyn_dims=None): + networks_on_disk: list[network.NetworkOnDisk] = [available_network_aliases.get(name, None) for name in names] + if any(x is None for x in networks_on_disk): + list_available_networks() + networks_on_disk: list[network.NetworkOnDisk] = [available_network_aliases.get(name, None) for name in names] + failed_to_load_networks = [] + recompile_model = maybe_recompile_model(names, te_multipliers) + + loaded_networks.clear() + diffuser_loaded.clear() + diffuser_scales.clear() + timer['load'] = 0 + t0 = time.time() + + for i, (network_on_disk, name) in enumerate(zip(networks_on_disk, names)): + net = None + if network_on_disk is not None: + shorthash = getattr(network_on_disk, 'shorthash', '').lower() + if debug: + shared.log.debug(f'Load network: type=LoRA name="{name}" file="{network_on_disk.filename}" hash="{shorthash}"') + try: + if recompile_model: + shared.compiled_model_state.lora_model.append(f"{name}:{te_multipliers[i] if te_multipliers else shared.opts.extra_networks_default_multiplier}") + if shared.opts.lora_force_diffusers or network_overrides.check_override(shorthash): # OpenVINO only works with Diffusers LoRa loading + net = load_diffusers(name, network_on_disk, lora_scale=te_multipliers[i] if te_multipliers else shared.opts.extra_networks_default_multiplier) + else: + net = load_network(name, network_on_disk) + if net is not None: + net.mentioned_name = name + network_on_disk.read_hash() + except Exception as e: + shared.log.error(f'Load network: type=LoRA file="{network_on_disk.filename}" {e}') + if debug: + errors.display(e, 'LoRA') + continue + if net is None: + failed_to_load_networks.append(name) + shared.log.error(f'Load network: type=LoRA name="{name}" detected={network_on_disk.sd_version if network_on_disk is not None else None} failed') + continue + shared.sd_model.embedding_db.load_diffusers_embedding(None, net.bundle_embeddings) + net.te_multiplier = te_multipliers[i] if te_multipliers else shared.opts.extra_networks_default_multiplier + net.unet_multiplier = unet_multipliers[i] if unet_multipliers else shared.opts.extra_networks_default_multiplier + net.dyn_dim = dyn_dims[i] if dyn_dims else shared.opts.extra_networks_default_multiplier + loaded_networks.append(net) + + while len(lora_cache) > shared.opts.lora_in_memory_limit: + name = next(iter(lora_cache)) + lora_cache.pop(name, None) + + if len(diffuser_loaded) > 0: + shared.log.debug(f'Load network: type=LoRA loaded={diffuser_loaded} available={shared.sd_model.get_list_adapters()} active={shared.sd_model.get_active_adapters()} scales={diffuser_scales}') + try: + shared.sd_model.set_adapters(adapter_names=diffuser_loaded, adapter_weights=diffuser_scales) + if shared.opts.lora_fuse_diffusers: + shared.sd_model.fuse_lora(adapter_names=diffuser_loaded, lora_scale=1.0, fuse_unet=True, fuse_text_encoder=True) # fuse uses fixed scale since later apply does the scaling + shared.sd_model.unload_lora_weights() + except Exception as e: + shared.log.error(f'Load network: type=LoRA {e}') + if debug: + errors.display(e, 'LoRA') + + if len(loaded_networks) > 0 and debug: + shared.log.debug(f'Load network: type=LoRA loaded={len(loaded_networks)} cache={list(lora_cache)}') + + devices.torch_gc() + + if recompile_model: + shared.log.info("Load network: type=LoRA recompiling model") + backup_lora_model = shared.compiled_model_state.lora_model + if 'Model' in shared.opts.cuda_compile: + shared.sd_model = sd_models_compile.compile_diffusers(shared.sd_model) + + shared.compiled_model_state.lora_model = backup_lora_model + if shared.opts.diffusers_offload_mode == "balanced": + sd_models.apply_balanced_offload(shared.sd_model) + t1 = time.time() + timer['load'] += t1 - t0 + +def set_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv], updown, ex_bias): + weights_backup = getattr(self, "network_weights_backup", None) + bias_backup = getattr(self, "network_bias_backup", None) + if weights_backup is None and bias_backup is None: + return + device = self.weight.device + with devices.inference_context(): + if weights_backup is not None: + if updown is not None: + if len(weights_backup.shape) == 4 and weights_backup.shape[1] == 9: + # inpainting model. zero pad updown to make channel[1] 4 to 9 + updown = torch.nn.functional.pad(updown, (0, 0, 0, 0, 0, 5)) # pylint: disable=not-callable + weights_backup = weights_backup.clone().to(device) + weights_backup += updown.to(weights_backup) + if getattr(self, "quant_type", None) in ['nf4', 'fp4']: + bnb = model_quant.load_bnb('Load network: type=LoRA', silent=True) + if bnb is not None: + self.weight = bnb.nn.Params4bit(weights_backup, quant_state=self.quant_state, quant_type=self.quant_type, blocksize=self.blocksize) + else: + self.weight.copy_(weights_backup, non_blocking=True) + else: + self.weight.copy_(weights_backup, non_blocking=True) + if hasattr(self, "qweight") and hasattr(self, "freeze"): + self.freeze() + if bias_backup is not None: + if ex_bias is not None: + bias_backup = bias_backup.clone() + ex_bias.to(weights_backup) + self.bias.copy_(bias_backup) + else: + self.bias = None + self.to(device) + + +def maybe_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv], wanted_names): # pylint: disable=W0613 + weights_backup = getattr(self, "network_weights_backup", None) + if weights_backup is None and wanted_names != (): # pylint: disable=C1803 + if getattr(self.weight, "quant_type", None) in ['nf4', 'fp4']: + bnb = model_quant.load_bnb('Load network: type=LoRA', silent=True) + if bnb is not None: + with devices.inference_context(): + weights_backup = bnb.functional.dequantize_4bit(self.weight, quant_state=self.weight.quant_state, quant_type=self.weight.quant_type, blocksize=self.weight.blocksize,) + self.quant_state = self.weight.quant_state + self.quant_type = self.weight.quant_type + self.blocksize = self.weight.blocksize + else: + weights_backup = self.weight.clone() + else: + weights_backup = self.weight.clone() + if shared.opts.lora_offload_backup and weights_backup is not None: + weights_backup = weights_backup.to(devices.cpu) + self.network_weights_backup = weights_backup + bias_backup = getattr(self, "network_bias_backup", None) + if bias_backup is None: + if getattr(self, 'bias', None) is not None: + bias_backup = self.bias.clone() + else: + bias_backup = None + if shared.opts.lora_offload_backup and bias_backup is not None: + bias_backup = bias_backup.to(devices.cpu) + self.network_bias_backup = bias_backup + + +def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv]): + """ + Applies the currently selected set of networks to the weights of torch layer self. + If weights already have this particular set of networks applied, does nothing. + If not, restores orginal weights from backup and alters weights according to networks. + """ + network_layer_name = getattr(self, 'network_layer_name', None) + if network_layer_name is None: + return + t0 = time.time() + current_names = getattr(self, "network_current_names", ()) + wanted_names = tuple((x.name, x.te_multiplier, x.unet_multiplier, x.dyn_dim) for x in loaded_networks) + if any([net.modules.get(network_layer_name, None) for net in loaded_networks]): # noqa: C419 # pylint: disable=R1729 + maybe_backup_weights(self, wanted_names) + if current_names != wanted_names: + for net in loaded_networks: + # default workflow where module is known and has weights + module = net.modules.get(network_layer_name, None) + if module is not None and hasattr(self, 'weight'): + try: + with devices.inference_context(): + weight = self.weight # calculate quant weights once + updown, ex_bias = module.calc_updown(weight) + set_weights(self, updown, ex_bias) + except RuntimeError as e: + extra_network_lora.errors[net.name] = extra_network_lora.errors.get(net.name, 0) + 1 + if debug: + module_name = net.modules.get(network_layer_name, None) + shared.log.error(f'LoRA apply weight name="{net.name}" module="{module_name}" layer="{network_layer_name}" {e}') + errors.display(e, 'LoRA') + raise RuntimeError('LoRA apply weight') from e + continue + if module is None: + continue + shared.log.warning(f'LoRA network="{net.name}" layer="{network_layer_name}" unsupported operation') + extra_network_lora.errors[net.name] = extra_network_lora.errors.get(net.name, 0) + 1 + if not loaded_networks: # restore from backup + t5 = time.time() + set_weights(self, None, None) + self.network_current_names = wanted_names + t1 = time.time() + timer['apply'] += t1 - t0 + +def network_load(): + sd_model = getattr(shared.sd_model, "pipe", shared.sd_model) # wrapped model compatiblility + for component_name in ['text_encoder','text_encoder_2', 'unet', 'transformer']: + component = getattr(sd_model, component_name, None) + if component is not None: + for _, module in component.named_modules(): + network_apply_weights(module) + + +def list_available_networks(): + t0 = time.time() + available_networks.clear() + available_network_aliases.clear() + forbidden_network_aliases.clear() + available_network_hash_lookup.clear() + forbidden_network_aliases.update({"none": 1, "Addams": 1}) + if not os.path.exists(shared.cmd_opts.lora_dir): + shared.log.warning(f'LoRA directory not found: path="{shared.cmd_opts.lora_dir}"') + + def add_network(filename): + if not os.path.isfile(filename): + return + name = os.path.splitext(os.path.basename(filename))[0] + name = name.replace('.', '_') + try: + entry = network.NetworkOnDisk(name, filename) + available_networks[entry.name] = entry + if entry.alias in available_network_aliases: + forbidden_network_aliases[entry.alias.lower()] = 1 + if shared.opts.lora_preferred_name == 'filename': + available_network_aliases[entry.name] = entry + else: + available_network_aliases[entry.alias] = entry + if entry.shorthash: + available_network_hash_lookup[entry.shorthash] = entry + except OSError as e: # should catch FileNotFoundError and PermissionError etc. + shared.log.error(f'LoRA: filename="{filename}" {e}') + + candidates = list(files_cache.list_files(shared.cmd_opts.lora_dir, ext_filter=[".pt", ".ckpt", ".safetensors"])) + with concurrent.futures.ThreadPoolExecutor(max_workers=shared.max_workers) as executor: + for fn in candidates: + executor.submit(add_network, fn) + t1 = time.time() + shared.log.info(f'Available LoRAs: path="{shared.cmd_opts.lora_dir}" items={len(available_networks)} folders={len(forbidden_network_aliases)} time={t1 - t0:.2f}') + + +def infotext_pasted(infotext, params): # pylint: disable=W0613 + if "AddNet Module 1" in [x[1] for x in scripts.scripts_txt2img.infotext_fields]: + return # if the other extension is active, it will handle those fields, no need to do anything + added = [] + for k in params: + if not k.startswith("AddNet Model "): + continue + num = k[13:] + if params.get("AddNet Module " + num) != "LoRA": + continue + name = params.get("AddNet Model " + num) + if name is None: + continue + m = re_network_name.match(name) + if m: + name = m.group(1) + multiplier = params.get("AddNet Weight A " + num, "1.0") + added.append(f"") + if added: + params["Prompt"] += "\n" + "".join(added) + + +list_available_networks() diff --git a/modules/lora/ui_extra_networks_lora.py b/modules/lora/ui_extra_networks_lora.py new file mode 100644 index 000000000..73cce47a3 --- /dev/null +++ b/modules/lora/ui_extra_networks_lora.py @@ -0,0 +1,123 @@ +import os +import json +import concurrent +import modules.lora.networks as networks +from modules import shared, ui_extra_networks + + +debug = os.environ.get('SD_LORA_DEBUG', None) is not None + + +class ExtraNetworksPageLora(ui_extra_networks.ExtraNetworksPage): + def __init__(self): + super().__init__('Lora') + self.list_time = 0 + + def refresh(self): + networks.list_available_networks() + + @staticmethod + def get_tags(l, info): + tags = {} + try: + if l.metadata is not None: + modelspec_tags = l.metadata.get('modelspec.tags', {}) + possible_tags = l.metadata.get('ss_tag_frequency', {}) # tags from model metedata + if isinstance(possible_tags, str): + possible_tags = {} + if isinstance(modelspec_tags, str): + modelspec_tags = {} + if len(list(modelspec_tags)) > 0: + possible_tags.update(modelspec_tags) + for k, v in possible_tags.items(): + words = k.split('_', 1) if '_' in k else [v, k] + words = [str(w).replace('.json', '') for w in words] + if words[0] == '{}': + words[0] = 0 + tag = ' '.join(words[1:]).lower() + tags[tag] = words[0] + + def find_version(): + found_versions = [] + current_hash = l.hash[:8].upper() + all_versions = info.get('modelVersions', []) + for v in info.get('modelVersions', []): + for f in v.get('files', []): + if any(h.startswith(current_hash) for h in f.get('hashes', {}).values()): + found_versions.append(v) + if len(found_versions) == 0: + found_versions = all_versions + return found_versions + + for v in find_version(): # trigger words from info json + possible_tags = v.get('trainedWords', []) + if isinstance(possible_tags, list): + for tag_str in possible_tags: + for tag in tag_str.split(','): + tag = tag.strip().lower() + if tag not in tags: + tags[tag] = 0 + + possible_tags = info.get('tags', []) # tags from info json + if not isinstance(possible_tags, list): + possible_tags = list(possible_tags.values()) + for tag in possible_tags: + tag = tag.strip().lower() + if tag not in tags: + tags[tag] = 0 + except Exception: + pass + bad_chars = [';', ':', '<', ">", "*", '?', '\'', '\"', '(', ')', '[', ']', '{', '}', '\\', '/'] + clean_tags = {} + for k, v in tags.items(): + tag = ''.join(i for i in k if i not in bad_chars).strip() + clean_tags[tag] = v + + clean_tags.pop('img', None) + clean_tags.pop('dataset', None) + return clean_tags + + def create_item(self, name): + l = networks.available_networks.get(name) + if l is None: + shared.log.warning(f'Networks: type=lora registered={len(list(networks.available_networks))} file="{name}" not registered') + return None + try: + # path, _ext = os.path.splitext(l.filename) + name = os.path.splitext(os.path.relpath(l.filename, shared.cmd_opts.lora_dir))[0] + item = { + "type": 'Lora', + "name": name, + "filename": l.filename, + "hash": l.shorthash, + "prompt": json.dumps(f" "), + "metadata": json.dumps(l.metadata, indent=4) if l.metadata else None, + "mtime": os.path.getmtime(l.filename), + "size": os.path.getsize(l.filename), + "version": l.sd_version, + } + info = self.find_info(l.filename) + item["info"] = info + item["description"] = self.find_description(l.filename, info) # use existing info instead of double-read + item["tags"] = self.get_tags(l, info) + return item + except Exception as e: + shared.log.error(f'Networks: type=lora file="{name}" {e}') + if debug: + from modules import errors + errors.display(e, 'Lora') + return None + + def list_items(self): + items = [] + with concurrent.futures.ThreadPoolExecutor(max_workers=shared.max_workers) as executor: + future_items = {executor.submit(self.create_item, net): net for net in networks.available_networks} + for future in concurrent.futures.as_completed(future_items): + item = future.result() + if item is not None: + items.append(item) + self.update_all_previews(items) + return items + + def allowed_directories_for_previews(self): + return [shared.cmd_opts.lora_dir, shared.cmd_opts.lyco_dir] diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 7b91fcd42..fe2d9407c 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -8,6 +8,8 @@ from modules.processing_helpers import resize_hires, calculate_base_steps, calculate_hires_steps, calculate_refiner_steps, save_intermediate, update_sampler, is_txt2img, is_refiner_enabled from modules.processing_args import set_pipeline_args from modules.onnx_impl import preprocess_pipeline as preprocess_onnx_pipeline, check_parameters_changed as olive_check_parameters_changed +from modules.lora.networks import network_load +from modules.lora.networks import timer as network_timer debug = shared.log.trace if os.environ.get('SD_DIFFUSERS_DEBUG', None) is not None else lambda *args, **kwargs: None @@ -425,6 +427,9 @@ def process_diffusers(p: processing.StableDiffusionProcessing): p.prompts = p.all_prompts[p.iteration * p.batch_size:(p.iteration+1) * p.batch_size] if p.negative_prompts is None or len(p.negative_prompts) == 0: p.negative_prompts = p.all_negative_prompts[p.iteration * p.batch_size:(p.iteration+1) * p.batch_size] + network_timer['apply'] = 0 + network_timer['restore'] = 0 + network_load() sd_models.move_model(shared.sd_model, devices.device) sd_models_compile.openvino_recompile_model(p, hires=False, refiner=False) # recompile if a parameter changes diff --git a/modules/shared.py b/modules/shared.py index 9ba98469e..7dead4dba 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -909,6 +909,7 @@ def get_default_modes(): "lora_in_memory_limit": OptionInfo(0, "LoRA memory cache", gr.Slider, {"minimum": 0, "maximum": 24, "step": 1}), "lora_quant": OptionInfo("NF4","LoRA precision in quantized models", gr.Radio, {"choices": ["NF4", "FP4"]}), "lora_load_gpu": OptionInfo(True if not cmd_opts.lowvram else False, "Load LoRA directly to GPU"), + "lora_offload_backup": OptionInfo(True, "Offload LoRA Backup Weights"), })) options_templates.update(options_section((None, "Internal options"), { diff --git a/scripts/lora_script.py b/scripts/lora_script.py new file mode 100644 index 000000000..a153a2caa --- /dev/null +++ b/scripts/lora_script.py @@ -0,0 +1,62 @@ +import re +import modules.lora.networks as networks +from modules.lora.lora_extract import create_ui +from modules.lora.network import NetworkOnDisk +from modules.lora.ui_extra_networks_lora import ExtraNetworksPageLora +from modules.lora.extra_networks_lora import ExtraNetworkLora +from modules import script_callbacks, extra_networks, ui_extra_networks, ui_models, shared # pylint: disable=unused-import + + +re_lora = re.compile(" Date: Sun, 24 Nov 2024 14:16:33 -0600 Subject: [PATCH 003/249] Enable stepwise LoRA (untested) --- modules/processing_callbacks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/processing_callbacks.py b/modules/processing_callbacks.py index 7d6c8ec04..0bb94abad 100644 --- a/modules/processing_callbacks.py +++ b/modules/processing_callbacks.py @@ -4,6 +4,7 @@ import torch import numpy as np from modules import shared, processing_correction, extra_networks, timer, prompt_parser_diffusers +from modules.lora.networks import network_load p = None debug = os.environ.get('SD_CALLBACK_DEBUG', None) is not None @@ -65,6 +66,7 @@ def diffusers_callback(pipe, step: int = 0, timestep: int = 0, kwargs: dict = {} time.sleep(0.1) if hasattr(p, "stepwise_lora"): extra_networks.activate(p, p.extra_network_data, step=step) + network_load() if latents is None: return kwargs elif shared.opts.nan_skip: From 4a2d999d73969526d1185fe673faecfb64a69173 Mon Sep 17 00:00:00 2001 From: AI-Casanova <54461896+AI-Casanova@users.noreply.github.com> Date: Sun, 24 Nov 2024 23:22:57 -0600 Subject: [PATCH 004/249] Fix multiple LoRA --- modules/lora/networks.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/modules/lora/networks.py b/modules/lora/networks.py index 762705b67..c6fde3e04 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -354,6 +354,8 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn if any([net.modules.get(network_layer_name, None) for net in loaded_networks]): # noqa: C419 # pylint: disable=R1729 maybe_backup_weights(self, wanted_names) if current_names != wanted_names: + batch_updown = None + batch_ex_bias = None for net in loaded_networks: # default workflow where module is known and has weights module = net.modules.get(network_layer_name, None) @@ -362,7 +364,14 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn with devices.inference_context(): weight = self.weight # calculate quant weights once updown, ex_bias = module.calc_updown(weight) - set_weights(self, updown, ex_bias) + if batch_updown is not None and updown is not None: + batch_updown += updown + else: + batch_updown = updown + if batch_ex_bias is not None and ex_bias is not None: + batch_ex_bias += ex_bias + else: + batch_ex_bias = ex_bias except RuntimeError as e: extra_network_lora.errors[net.name] = extra_network_lora.errors.get(net.name, 0) + 1 if debug: @@ -375,9 +384,7 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn continue shared.log.warning(f'LoRA network="{net.name}" layer="{network_layer_name}" unsupported operation') extra_network_lora.errors[net.name] = extra_network_lora.errors.get(net.name, 0) + 1 - if not loaded_networks: # restore from backup - t5 = time.time() - set_weights(self, None, None) + set_weights(self, batch_updown, batch_ex_bias) # Set or restore weights from backup self.network_current_names = wanted_names t1 = time.time() timer['apply'] += t1 - t0 From 38f467ab132cda1def68989c01ed29c0ed6981ec Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 28 Nov 2024 09:11:42 -0500 Subject: [PATCH 005/249] euler flowmatch add sigma methods Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 3 ++- modules/sd_samplers_diffusers.py | 20 ++++++++++---------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3f2282c0..f88c78302 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,7 +50,8 @@ - control: add stats - browser->server logging framework - **Sampler** improvements - - update DPM FlowMatch samplers + - Euler FlowMatch: add sigma methods (*karras/exponential/betas*) + - DPM FlowMatch: update all and add sigma methods ### Fixes diff --git a/modules/sd_samplers_diffusers.py b/modules/sd_samplers_diffusers.py index 9f24d5a91..4672df92e 100644 --- a/modules/sd_samplers_diffusers.py +++ b/modules/sd_samplers_diffusers.py @@ -69,7 +69,7 @@ 'Euler a': { 'steps_offset': 0, 'rescale_betas_zero_snr': False, 'timestep_spacing': 'linspace' }, 'Euler SGM': { 'steps_offset': 0, 'interpolation_type': "linear", 'rescale_betas_zero_snr': False, 'final_sigmas_type': 'zero', 'timestep_spacing': 'trailing', 'use_beta_sigmas': False, 'use_exponential_sigmas': False, 'use_karras_sigmas': False, 'prediction_type': "sample" }, 'Euler EDM': { 'sigma_schedule': "karras" }, - 'Euler FlowMatch': { 'timestep_spacing': "linspace", 'shift': 1, 'use_dynamic_shifting': False }, + 'Euler FlowMatch': { 'timestep_spacing': "linspace", 'shift': 1, 'use_dynamic_shifting': False, 'use_karras_sigmas': False, 'use_exponential_sigmas': False, 'use_beta_sigmas': False }, 'DPM++': { 'solver_order': 2, 'thresholding': False, 'sample_max_value': 1.0, 'algorithm_type': "dpmsolver++", 'solver_type': "midpoint", 'lower_order_final': True, 'use_karras_sigmas': False, 'use_exponential_sigmas': False, 'use_beta_sigmas': False, 'final_sigmas_type': 'sigma_min' }, 'DPM++ 1S': { 'thresholding': False, 'sample_max_value': 1.0, 'algorithm_type': "dpmsolver++", 'solver_type': "midpoint", 'lower_order_final': True, 'use_karras_sigmas': False, 'use_exponential_sigmas': False, 'use_beta_sigmas': False, 'use_lu_lambdas': False, 'final_sigmas_type': 'zero', 'timestep_spacing': 'linspace', 'solver_order': 1 }, @@ -200,16 +200,16 @@ def __init__(self, name, constructor, model, **kwargs): timesteps = re.split(',| ', shared.opts.schedulers_timesteps) timesteps = [int(x) for x in timesteps if x.isdigit()] if len(timesteps) == 0: - if 'use_beta_sigmas' in self.config: - self.config['use_beta_sigmas'] = shared.opts.schedulers_sigma == 'beta' - if 'use_karras_sigmas' in self.config: - self.config['use_karras_sigmas'] = shared.opts.schedulers_sigma == 'karras' - if 'use_exponential_sigmas' in self.config: - self.config['use_exponential_sigmas'] = shared.opts.schedulers_sigma == 'exponential' - if 'use_lu_lambdas' in self.config: - self.config['use_lu_lambdas'] = shared.opts.schedulers_sigma == 'lambdas' if 'sigma_schedule' in self.config: self.config['sigma_schedule'] = shared.opts.schedulers_sigma if shared.opts.schedulers_sigma != 'default' else None + if shared.opts.schedulers_sigma == 'betas' and 'use_beta_sigmas' in self.config: + self.config['use_beta_sigmas'] = True + elif shared.opts.schedulers_sigma == 'karras' and 'use_karras_sigmas' in self.config: + self.config['use_karras_sigmas'] = True + elif shared.opts.schedulers_sigma == 'exponential' and 'use_exponential_sigmas' in self.config: + self.config['use_exponential_sigmas'] = True + elif shared.opts.schedulers_sigma == 'lambdas' and 'use_lu_lambdas' in self.config: + self.config['use_lu_lambdas'] = True else: pass # timesteps are set using set_timesteps in set_pipeline_args @@ -236,7 +236,7 @@ def __init__(self, name, constructor, model, **kwargs): if 'use_dynamic_shifting' in self.config: if 'Flux' in model.__class__.__name__: self.config['use_dynamic_shifting'] = shared.opts.schedulers_dynamic_shift - if 'use_beta_sigmas' in self.config: + if 'use_beta_sigmas' in self.config and 'sigma_schedule' in self.config: self.config['use_beta_sigmas'] = 'StableDiffusion3' in model.__class__.__name__ if 'rescale_betas_zero_snr' in self.config: self.config['rescale_betas_zero_snr'] = shared.opts.schedulers_rescale_betas From e519dcaa751cf44eaa462aa8eae492415a551525 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 28 Nov 2024 13:28:46 -0500 Subject: [PATCH 006/249] simplify impaint/sketch canvas handling Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 5 ++- javascript/black-teal.css | 1 - javascript/imageMaskFix.js | 38 ------------------ javascript/sdnext.css | 20 ++++++---- javascript/ui.js | 2 +- modules/img2img.py | 14 +++---- modules/ui_control.py | 4 +- modules/ui_img2img.py | 81 ++++++++++++++++++-------------------- 8 files changed, 65 insertions(+), 100 deletions(-) delete mode 100644 javascript/imageMaskFix.js diff --git a/CHANGELOG.md b/CHANGELOG.md index f88c78302..919041bde 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,7 +48,8 @@ - control: hide preview column by default - control: optionn to hide input column - control: add stats - - browser->server logging framework + - browser -> server logging framework + - add addtional themes: `black-reimagined` - **Sampler** improvements - Euler FlowMatch: add sigma methods (*karras/exponential/betas*) - DPM FlowMatch: update all and add sigma methods @@ -65,6 +66,8 @@ - allow xyz-grid with multi-axis s&r - fix xyz-grid with lora - fix api script callbacks +- fix gpu memory monitoring +- simplify img2img/inpaint/sketch canvas handling ## Update for 2024-11-21 diff --git a/javascript/black-teal.css b/javascript/black-teal.css index b73f9fdc7..2ebf32e96 100644 --- a/javascript/black-teal.css +++ b/javascript/black-teal.css @@ -108,7 +108,6 @@ fieldset .gr-block.gr-box, label.block span { padding: 0; margin-top: -4px; } .eta-bar { display: none !important } .gradio-slider { max-width: 200px; } .gradio-slider input[type="number"] { background: var(--neutral-950); margin-top: 2px; } -.gradio-image { height: unset !important; } svg.feather.feather-image, .feather .feather-image { display: none } .gap-2 { padding-top: 8px; } .gr-box > div > div > input.gr-text-input { right: 0; width: 4em; padding: 0; top: -12px; border: none; max-height: 20px; } diff --git a/javascript/imageMaskFix.js b/javascript/imageMaskFix.js deleted file mode 100644 index fd37caf90..000000000 --- a/javascript/imageMaskFix.js +++ /dev/null @@ -1,38 +0,0 @@ -/** - * temporary fix for https://github.com/AUTOMATIC1111/stable-diffusion-webui/issues/668 - * @see https://github.com/gradio-app/gradio/issues/1721 - */ -function imageMaskResize() { - const canvases = gradioApp().querySelectorAll('#img2maskimg .touch-none canvas'); - if (!canvases.length) { - window.removeEventListener('resize', imageMaskResize); - return; - } - const wrapper = canvases[0].closest('.touch-none'); - const previewImage = wrapper.previousElementSibling; - if (!previewImage.complete) { - previewImage.addEventListener('load', imageMaskResize); - return; - } - const w = previewImage.width; - const h = previewImage.height; - const nw = previewImage.naturalWidth; - const nh = previewImage.naturalHeight; - const portrait = nh > nw; - const wW = Math.min(w, portrait ? h / nh * nw : w / nw * nw); - const wH = Math.min(h, portrait ? h / nh * nh : w / nw * nh); - wrapper.style.width = `${wW}px`; - wrapper.style.height = `${wH}px`; - wrapper.style.left = '0px'; - wrapper.style.top = '0px'; - canvases.forEach((c) => { - c.style.width = ''; - c.style.height = ''; - c.style.maxWidth = '100%'; - c.style.maxHeight = '100%'; - c.style.objectFit = 'contain'; - }); -} - -onAfterUiUpdate(imageMaskResize); -window.addEventListener('resize', imageMaskResize); diff --git a/javascript/sdnext.css b/javascript/sdnext.css index 240b7492f..c5145c973 100644 --- a/javascript/sdnext.css +++ b/javascript/sdnext.css @@ -30,6 +30,19 @@ input::-webkit-outer-spin-button, input::-webkit-inner-spin-button { margin-left .hidden { display: none; } .tabitem { padding: 0 !important; } +/* gradio image/canvas elements */ +.image-container { overflow: auto; } +/* +.gradio-image { min-height: fit-content; } +.gradio-image img { object-fit: contain; } +*/ +/* +.gradio-image { min-height: 200px !important; } +.image-container { height: unset !important; } +.control-image { height: unset !important; } +#img2img_sketch, #img2maskimg, #inpaint_sketch { overflow: overlay !important; resize: auto; background: var(--panel-background-fill); z-index: 5; } +*/ + /* color elements */ .gradio-dropdown, .block.gradio-slider, .block.gradio-checkbox, .block.gradio-textbox, .block.gradio-radio, .block.gradio-checkboxgroup, .block.gradio-number, .block.gradio-colorpicker { border-width: 0 !important; box-shadow: none !important;} .gradio-accordion { padding-top: var(--spacing-md) !important; padding-right: 0 !important; padding-bottom: 0 !important; color: var(--body-text-color); } @@ -87,8 +100,6 @@ button.custom-button { border-radius: var(--button-large-radius); padding: var(- .performance .time { margin-right: 0; } .thumbnails { background: var(--body-background-fill); } .prompt textarea { resize: vertical; } -.image-container { height: unset !important; } -.control-image { height: unset !important; } .grid-wrap { overflow-y: auto !important; } #control_results { margin: 0; padding: 0; } #txt2img_gallery, #img2img_gallery { height: 50vh; } @@ -106,7 +117,6 @@ button.custom-button { border-radius: var(--button-large-radius); padding: var(- #txt2img_prompt, #txt2img_neg_prompt, #img2img_prompt, #img2img_neg_prompt, #control_prompt, #control_neg_prompt { display: contents; } #txt2img_actions_column, #img2img_actions_column, #control_actions { flex-flow: wrap; justify-content: space-between; } - .interrogate-clip { position: absolute; right: 6em; top: 8px; max-width: fit-content; background: none !important; z-index: 50; } .interrogate-blip { position: absolute; right: 4em; top: 8px; max-width: fit-content; background: none !important; z-index: 50; } .interrogate-col { min-width: 0 !important; max-width: fit-content; margin-right: var(--spacing-xxl); } @@ -119,8 +129,6 @@ div#extras_scale_to_tab div.form { flex-direction: row; } #img2img_unused_scale_by_slider { visibility: hidden; width: 0.5em; max-width: 0.5em; min-width: 0.5em; } .inactive{ opacity: 0.5; } div#extras_scale_to_tab div.form { flex-direction: row; } -#mode_img2img .gradio-image>div.fixed-height, #mode_img2img .gradio-image>div.fixed-height img{ height: 480px !important; max-height: 480px !important; min-height: 480px !important; } -#img2img_sketch, #img2maskimg, #inpaint_sketch { overflow: overlay !important; resize: auto; background: var(--panel-background-fill); z-index: 5; } .image-buttons button { min-width: auto; } .infotext { overflow-wrap: break-word; line-height: 1.5em; font-size: 0.95em !important; } .infotext > p { white-space: pre-wrap; color: var(--block-info-text-color) !important; } @@ -380,8 +388,6 @@ div:has(>#tab-gallery-folders) { flex-grow: 0 !important; background-color: var( #img2img_actions_column { display: flex; min-width: fit-content !important; flex-direction: row;justify-content: space-evenly; align-items: center;} #txt2img_generate_box, #img2img_generate_box, #txt2img_enqueue_wrapper,#img2img_enqueue_wrapper {display: flex;flex-direction: column;height: 4em !important;align-items: stretch;justify-content: space-evenly;} #img2img_interface, #img2img_results, #img2img_footer p { text-wrap: wrap; min-width: 100% !important; max-width: 100% !important;} /* maintain single column for from image operations on larger mobile devices */ - #img2img_sketch, #img2maskimg, #inpaint_sketch {display: flex; overflow: auto !important; resize: none !important; } /* fix inpaint image display being too large for mobile displays */ - #img2maskimg canvas { width: auto !important; max-height: 100% !important; height: auto !important; } #txt2img_sampler, #txt2img_batch, #txt2img_seed_group, #txt2img_advanced, #txt2img_second_pass, #img2img_sampling_group, #img2img_resize_group, #img2img_batch_group, #img2img_seed_group, #img2img_denoise_group, #img2img_advanced_group { width: 100% !important; } /* fix from text/image UI elements to prevent them from moving around within the UI */ #img2img_resize_group .gradio-radio>div { display: flex; flex-direction: column; width: unset !important; } #inpaint_controls div { display:flex;flex-direction: row;} diff --git a/javascript/ui.js b/javascript/ui.js index 81d1c67e4..3e3f14390 100644 --- a/javascript/ui.js +++ b/javascript/ui.js @@ -139,7 +139,7 @@ function switch_to_inpaint(...args) { return Array.from(arguments); } -function switch_to_inpaint_sketch(...args) { +function switch_to_composite(...args) { switchToTab('Image'); switch_to_img2img_tab(3); return Array.from(arguments); diff --git a/modules/img2img.py b/modules/img2img.py index 8274386cc..077df1259 100644 --- a/modules/img2img.py +++ b/modules/img2img.py @@ -164,12 +164,7 @@ def img2img(id_task: str, state: str, mode: int, return [], '', '', 'Error: init image not provided' image = init_img.convert("RGB") mask = None - elif mode == 1: # img2img sketch - if sketch is None: - return [], '', '', 'Error: sketch image not provided' - image = sketch.convert("RGB") - mask = None - elif mode == 2: # inpaint + elif mode == 1: # inpaint if init_img_with_mask is None: return [], '', '', 'Error: init image with mask not provided' image = init_img_with_mask["image"] @@ -177,7 +172,12 @@ def img2img(id_task: str, state: str, mode: int, alpha_mask = ImageOps.invert(image.split()[-1]).convert('L').point(lambda x: 255 if x > 0 else 0, mode='1') mask = ImageChops.lighter(alpha_mask, mask.convert('L')).convert('L') image = image.convert("RGB") - elif mode == 3: # inpaint sketch + elif mode == 2: # sketch + if sketch is None: + return [], '', '', 'Error: sketch image not provided' + image = sketch.convert("RGB") + mask = None + elif mode == 3: # composite if inpaint_color_sketch is None: return [], '', '', 'Error: color sketch image not provided' image = inpaint_color_sketch diff --git a/modules/ui_control.py b/modules/ui_control.py index f4329663a..072d9b9c9 100644 --- a/modules/ui_control.py +++ b/modules/ui_control.py @@ -13,7 +13,7 @@ from modules import ui_control_helpers as helpers -gr_height = None +gr_height = 512 max_units = shared.opts.control_max_units units: list[unit.Unit] = [] # main state variable controls: list[gr.component] = [] # list of gr controls @@ -135,7 +135,7 @@ def create_ui(_blocks: gr.Blocks=None): with gr.Row(): input_type = gr.Radio(label="Input type", choices=['Control only', 'Init image same as control', 'Separate init image'], value='Control only', type='index', elem_id='control_input_type') with gr.Row(): - denoising_strength = gr.Slider(minimum=0.01, maximum=1.0, step=0.01, label='Denoising strength', value=0.50, elem_id="control_input_denoising_strength") + denoising_strength = gr.Slider(minimum=0.01, maximum=1.0, step=0.01, label='Denoising strength', value=0.30, elem_id="control_input_denoising_strength") with gr.Accordion(open=False, label="Size", elem_id="control_size", elem_classes=["small-accordion"]): with gr.Tabs(): diff --git a/modules/ui_img2img.py b/modules/ui_img2img.py index 046c181ce..3c3d63656 100644 --- a/modules/ui_img2img.py +++ b/modules/ui_img2img.py @@ -1,7 +1,6 @@ import os from PIL import Image import gradio as gr -import numpy as np from modules.call_queue import wrap_gradio_gpu_call, wrap_queued_call from modules import timer, shared, ui_common, ui_sections, generation_parameters_copypaste, processing_vae @@ -56,7 +55,7 @@ def copy_image(img): def add_copy_image_controls(tab_name, elem): with gr.Row(variant="compact", elem_id=f"img2img_copy_to_{tab_name}"): - for title, name in zip(['➠ Image', '➠ Sketch', '➠ Inpaint', '➠ Composite'], ['img2img', 'sketch', 'inpaint', 'inpaint_sketch']): + for title, name in zip(['➠ Image', '➠ Inpaint', '➠ Sketch', '➠ Composite'], ['img2img', 'sketch', 'inpaint', 'composite']): if name == tab_name: gr.Button(title, elem_id=f'copy_to_{name}', interactive=False) copy_image_destinations[name] = elem @@ -67,33 +66,36 @@ def add_copy_image_controls(tab_name, elem): with gr.Tabs(elem_id="mode_img2img"): img2img_selected_tab = gr.State(0) # pylint: disable=abstract-class-instantiated state = gr.Textbox(value='', visible=False) - with gr.TabItem('Image', id='img2img', elem_id="img2img_img2img_tab") as tab_img2img: - init_img = gr.Image(label="Image for img2img", elem_id="img2img_image", show_label=False, source="upload", interactive=True, type="pil", tool="editor", image_mode="RGBA") + with gr.TabItem('Image', id='img2img_image', elem_id="img2img_image_tab") as tab_img2img: + img_init = gr.Image(label="", elem_id="img2img_image", show_label=False, source="upload", interactive=True, type="pil", tool="editor", image_mode="RGBA", height=512) interrogate_clip, interrogate_booru = ui_sections.create_interrogate_buttons('img2img') - add_copy_image_controls('img2img', init_img) - - with gr.TabItem('Sketch', id='img2img_sketch', elem_id="img2img_img2img_sketch_tab") as tab_sketch: - sketch = gr.Image(label="Image for img2img", elem_id="img2img_sketch", show_label=False, source="upload", interactive=True, type="pil", tool="color-sketch", image_mode="RGBA") - add_copy_image_controls('sketch', sketch) - - with gr.TabItem('Inpaint', id='inpaint', elem_id="img2img_inpaint_tab") as tab_inpaint: - init_img_with_mask = gr.Image(label="Image for inpainting with mask", show_label=False, elem_id="img2maskimg", source="upload", interactive=True, type="pil", tool="sketch", image_mode="RGBA") - add_copy_image_controls('inpaint', init_img_with_mask) - - with gr.TabItem('Composite', id='inpaint_sketch', elem_id="img2img_inpaint_sketch_tab") as tab_inpaint_color: - inpaint_color_sketch = gr.Image(label="Color sketch inpainting", show_label=False, elem_id="inpaint_sketch", source="upload", interactive=True, type="pil", tool="color-sketch", image_mode="RGBA") - inpaint_color_sketch_orig = gr.State(None) # pylint: disable=abstract-class-instantiated - add_copy_image_controls('inpaint_sketch', inpaint_color_sketch) - - def update_orig(image, state): - if image is not None: - same_size = state is not None and state.size == image.size - has_exact_match = np.any(np.all(np.array(image) == np.array(state), axis=-1)) - edited = same_size and has_exact_match - return image if not edited or state is None else state - return state - - inpaint_color_sketch.change(update_orig, [inpaint_color_sketch, inpaint_color_sketch_orig], inpaint_color_sketch_orig) + add_copy_image_controls('img2img', img_init) + + with gr.TabItem('Inpaint', id='img2img_inpaint', elem_id="img2img_inpaint_tab") as tab_inpaint: + img_inpaint = gr.Image(label="", elem_id="img2img_inpaint", show_label=False, source="upload", interactive=True, type="pil", tool="sketch", image_mode="RGBA", height=512) + add_copy_image_controls('inpaint', img_inpaint) + + with gr.TabItem('Sketch', id='img2img_sketch', elem_id="img2img_sketch_tab") as tab_sketch: + img_sketch = gr.Image(label="", elem_id="img2img_sketch", show_label=False, source="upload", interactive=True, type="pil", tool="color-sketch", image_mode="RGBA", height=512) + add_copy_image_controls('sketch', img_sketch) + + with gr.TabItem('Composite', id='img2img_composite', elem_id="img2img_composite_tab") as tab_inpaint_color: + img_composite = gr.Image(label="", show_label=False, elem_id="img2img_composite", source="upload", interactive=True, type="pil", tool="color-sketch", image_mode="RGBA", height=512) + img_composite_orig = gr.State(None) # pylint: disable=abstract-class-instantiated + img_composite_orig_update = False + + def fn_img_composite_upload(): + nonlocal img_composite_orig_update + img_composite_orig_update = True + def fn_img_composite_change(img, img_composite): + nonlocal img_composite_orig_update + res = img if img_composite_orig_update else img_composite + img_composite_orig_update = False + return res + + img_composite.upload(fn=fn_img_composite_upload, inputs=[], outputs=[]) + img_composite.change(fn=fn_img_composite_change, inputs=[img_composite, img_composite_orig], outputs=[img_composite_orig]) + add_copy_image_controls('composite', img_composite) with gr.TabItem('Upload', id='inpaint_upload', elem_id="img2img_inpaint_upload_tab") as tab_inpaint_upload: init_img_inpaint = gr.Image(label="Image for img2img", show_label=False, source="upload", interactive=True, type="pil", elem_id="img_inpaint_base") @@ -120,13 +122,13 @@ def update_orig(image, state): with gr.Accordion(open=False, label="Sampler", elem_classes=["small-accordion"], elem_id="img2img_sampler_group"): steps, sampler_index = ui_sections.create_sampler_and_steps_selection(None, "img2img") ui_sections.create_sampler_options('img2img') - resize_mode, resize_name, resize_context, width, height, scale_by, selected_scale_tab = ui_sections.create_resize_inputs('img2img', [init_img, sketch], latent=True, non_zero=False) + resize_mode, resize_name, resize_context, width, height, scale_by, selected_scale_tab = ui_sections.create_resize_inputs('img2img', [img_init, img_sketch], latent=True, non_zero=False) batch_count, batch_size = ui_sections.create_batch_inputs('img2img', accordion=True) seed, reuse_seed, subseed, reuse_subseed, subseed_strength, seed_resize_from_h, seed_resize_from_w = ui_sections.create_seed_inputs('img2img') with gr.Accordion(open=False, label="Denoise", elem_classes=["small-accordion"], elem_id="img2img_denoise_group"): with gr.Row(): - denoising_strength = gr.Slider(minimum=0.0, maximum=0.99, step=0.01, label='Denoising strength', value=0.50, elem_id="img2img_denoising_strength") + denoising_strength = gr.Slider(minimum=0.0, maximum=0.99, step=0.01, label='Denoising strength', value=0.30, elem_id="img2img_denoising_strength") refiner_start = gr.Slider(minimum=0.0, maximum=1.0, step=0.05, label='Denoise start', value=0.0, elem_id="img2img_refiner_start") full_quality, tiling, hidiffusion, cfg_scale, clip_skip, image_cfg_scale, diffusers_guidance_rescale, pag_scale, pag_adaptive, cfg_end = ui_sections.create_advanced_inputs('img2img') @@ -167,13 +169,8 @@ def select_img2img_tab(tab): img2img_args = [ dummy_component1, state, dummy_component2, img2img_prompt, img2img_negative_prompt, img2img_prompt_styles, - init_img, - sketch, - init_img_with_mask, - inpaint_color_sketch, - inpaint_color_sketch_orig, - init_img_inpaint, - init_mask_inpaint, + img_init, img_sketch, img_inpaint, img_composite, img_composite_orig, + init_img_inpaint, init_mask_inpaint, steps, sampler_index, mask_blur, mask_alpha, @@ -225,10 +222,7 @@ def select_img2img_tab(tab): img2img_batch_files, img2img_batch_input_dir, img2img_batch_output_dir, - init_img, - sketch, - init_img_with_mask, - inpaint_color_sketch, + img_init, img_sketch, img_inpaint, img_composite, init_img_inpaint, ], outputs=[img2img_prompt, dummy_component], @@ -285,7 +279,8 @@ def select_img2img_tab(tab): (seed_resize_from_h, "Seed resize from-2"), *modules.scripts.scripts_img2img.infotext_fields ] - generation_parameters_copypaste.add_paste_fields("img2img", init_img, img2img_paste_fields, override_settings) - generation_parameters_copypaste.add_paste_fields("inpaint", init_img_with_mask, img2img_paste_fields, override_settings) + generation_parameters_copypaste.add_paste_fields("img2img", img_init, img2img_paste_fields, override_settings) + generation_parameters_copypaste.add_paste_fields("sketch", img_sketch, img2img_paste_fields, override_settings) + generation_parameters_copypaste.add_paste_fields("inpaint", img_inpaint, img2img_paste_fields, override_settings) img2img_bindings = generation_parameters_copypaste.ParamBinding(paste_button=img2img_paste, tabname="img2img", source_text_component=img2img_prompt, source_image_component=None) generation_parameters_copypaste.register_paste_params_button(img2img_bindings) From 4524f751ec56cb7d5641af7e513ede17dbc4aa3a Mon Sep 17 00:00:00 2001 From: Disty0 Date: Thu, 28 Nov 2024 22:12:25 +0300 Subject: [PATCH 007/249] Fix Cascade and add full_vqgan_decode --- modules/processing_args.py | 2 +- modules/processing_diffusers.py | 2 +- modules/processing_vae.py | 64 ++++++++++++++++++++++++++++++++- 3 files changed, 65 insertions(+), 3 deletions(-) diff --git a/modules/processing_args.py b/modules/processing_args.py index ff766ec04..a716b685e 100644 --- a/modules/processing_args.py +++ b/modules/processing_args.py @@ -135,7 +135,7 @@ def set_pipeline_args(p, model, prompts: list, negative_prompts: list, prompts_2 prompts = [p.replace('|image|', '<|image_1|>') for p in prompts] if hasattr(model, 'text_encoder') and hasattr(model, 'tokenizer') and 'prompt_embeds' in possible and prompt_parser_diffusers.embedder is not None: args['prompt_embeds'] = prompt_parser_diffusers.embedder('prompt_embeds') - if 'StableCascade' in model.__class__.__name__ and len(getattr(p, 'negative_pooleds', [])) > 0: + if 'StableCascade' in model.__class__.__name__ and prompt_parser_diffusers.embedder is not None: args['prompt_embeds_pooled'] = prompt_parser_diffusers.embedder('positive_pooleds').unsqueeze(0) elif 'XL' in model.__class__.__name__ and prompt_parser_diffusers.embedder is not None: args['pooled_prompt_embeds'] = prompt_parser_diffusers.embedder('positive_pooleds') diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index fe2d9407c..2e8fb357c 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -354,7 +354,7 @@ def process_decode(p: processing.StableDiffusionProcessing, output): if not hasattr(model, 'vae'): if hasattr(model, 'pipe') and hasattr(model.pipe, 'vae'): model = model.pipe - if hasattr(model, "vae") and output.images is not None and len(output.images) > 0: + if (hasattr(model, "vae") or hasattr(model, "vqgan")) and output.images is not None and len(output.images) > 0: if p.hr_resize_mode > 0 and (p.hr_upscaler != 'None' or p.hr_resize_mode == 5): width = max(getattr(p, 'width', 0), getattr(p, 'hr_upscale_to_x', 0)) height = max(getattr(p, 'height', 0), getattr(p, 'hr_upscale_to_y', 0)) diff --git a/modules/processing_vae.py b/modules/processing_vae.py index 3c0357c81..1c4a45f07 100644 --- a/modules/processing_vae.py +++ b/modules/processing_vae.py @@ -33,6 +33,62 @@ def create_latents(image, p, dtype=None, device=None): return latents +def full_vqgan_decode(latents, model): + t0 = time.time() + if model is None or not hasattr(model, 'vqgan'): + shared.log.error('VQGAN not found in model') + return [] + if debug: + devices.torch_gc(force=True) + shared.mem_mon.reset() + + base_device = None + if shared.opts.diffusers_move_unet and not getattr(model, 'has_accelerate', False): + base_device = sd_models.move_base(model, devices.cpu) + + if shared.opts.diffusers_offload_mode == "balanced": + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) + elif shared.opts.diffusers_offload_mode != "sequential": + sd_models.move_model(model.vqgan, devices.device) + + latents = latents.to(devices.device, dtype=model.vqgan.dtype) + + #normalize latents + scaling_factor = model.vqgan.config.get("scale_factor", None) + if scaling_factor: + latents = latents * scaling_factor + + vae_name = os.path.splitext(os.path.basename(sd_vae.loaded_vae_file))[0] if sd_vae.loaded_vae_file is not None else "default" + vae_stats = f'name="{vae_name}" dtype={model.vqgan.dtype} device={model.vqgan.device}' + latents_stats = f'shape={latents.shape} dtype={latents.dtype} device={latents.device}' + stats = f'vae {vae_stats} latents {latents_stats}' + + log_debug(f'VAE config: {model.vqgan.config}') + try: + decoded = model.vqgan.decode(latents).sample.clamp(0, 1) + except Exception as e: + shared.log.error(f'VAE decode: {stats} {e}') + errors.display(e, 'VAE decode') + decoded = [] + + # delete vae after OpenVINO compile + if 'VAE' in shared.opts.cuda_compile and shared.opts.cuda_compile_backend == "openvino_fx" and shared.compiled_model_state.first_pass_vae: + shared.compiled_model_state.first_pass_vae = False + if not shared.opts.openvino_disable_memory_cleanup and hasattr(shared.sd_model, "vqgan"): + model.vqgan.apply(sd_models.convert_to_faketensors) + devices.torch_gc(force=True) + + if shared.opts.diffusers_offload_mode == "balanced": + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) + elif shared.opts.diffusers_move_unet and not getattr(model, 'has_accelerate', False) and base_device is not None: + sd_models.move_base(model, base_device) + t1 = time.time() + if debug: + log_debug(f'VAE memory: {shared.mem_mon.read()}') + shared.log.debug(f'VAE decode: {stats} time={round(t1-t0, 3)}') + return decoded + + def full_vae_decode(latents, model): t0 = time.time() if not hasattr(model, 'vae') and hasattr(model, 'pipe'): @@ -161,7 +217,7 @@ def vae_decode(latents, model, output_type='np', full_quality=True, width=None, return [] if shared.state.interrupted or shared.state.skipped: return [] - if not hasattr(model, 'vae'): + if not hasattr(model, 'vae') and not hasattr(model, 'vqgan'): shared.log.error('VAE not found in model') return [] @@ -176,12 +232,18 @@ def vae_decode(latents, model, output_type='np', full_quality=True, width=None, decoded = latents.float().cpu().numpy() elif full_quality and hasattr(model, "vae"): decoded = full_vae_decode(latents=latents, model=model) + elif hasattr(model, "vqgan"): + decoded = full_vqgan_decode(latents=latents, model=model) else: decoded = taesd_vae_decode(latents=latents) if torch.is_tensor(decoded): if hasattr(model, 'image_processor'): imgs = model.image_processor.postprocess(decoded, output_type=output_type) + elif hasattr(model, "vqgan"): + imgs = decoded.permute(0, 2, 3, 1).cpu().float().numpy() + if output_type == "pil": + imgs = model.numpy_to_pil(imgs) else: import diffusers model.image_processor = diffusers.image_processor.VaeImageProcessor() From 27e9d9e5f0ddeef46f892976ddd3b6e2f4835793 Mon Sep 17 00:00:00 2001 From: P-Hellmann Date: Fri, 29 Nov 2024 08:58:22 +0100 Subject: [PATCH 008/249] Small changes to black-teal-reimagined --- javascript/black-teal-reimagined.css | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/javascript/black-teal-reimagined.css b/javascript/black-teal-reimagined.css index e5618c02c..94fccdea9 100644 --- a/javascript/black-teal-reimagined.css +++ b/javascript/black-teal-reimagined.css @@ -813,10 +813,18 @@ textarea[rows="1"] { background: var(--background-color); box-shadow: var(--shadow-md); border-radius: var(--radius-lg); + transform: translateX(100%); + animation: slideIn 0.5s forwards; overflow: hidden; /* Prevents overflow of content */ } +@keyframes slideIn { + to { + transform: translateX(0); + } +} + /* Extra Networks Styles */ .extra-networks { background: var(--background-color); @@ -1032,6 +1040,20 @@ textarea[rows="1"] { height: 100%; } +/* Token counters styling */ + +#txt2img_token_counter, #txt2img_negative_token_counter { + display: flex; + flex-direction: column; + justify-content: space-evenly; + padding: 10px; +} + +#txt2img_prompt_container { + margin: 5px; + padding: 0px; +} + /* Based on Gradio Built-in Dark Theme */ :root, .light, From ad38d8a455c4ceee712948ac91a7edeeee867510 Mon Sep 17 00:00:00 2001 From: P-Hellmann Date: Fri, 29 Nov 2024 10:01:53 +0100 Subject: [PATCH 009/249] Removed redundant css --- javascript/black-teal-reimagined.css | 36 ++++------------------------ 1 file changed, 4 insertions(+), 32 deletions(-) diff --git a/javascript/black-teal-reimagined.css b/javascript/black-teal-reimagined.css index 94fccdea9..70315782d 100644 --- a/javascript/black-teal-reimagined.css +++ b/javascript/black-teal-reimagined.css @@ -655,27 +655,6 @@ svg.feather.feather-image, font-weight: normal; } -#txt2img_prompt, -#txt2img_neg_prompt, -#img2img_prompt, -#img2img _neg_prompt, -#control_prompt, -#control_neg_prompt { - background-color: var(--background-color); - box-shadow: none !important; -} - -#txt2img_prompt>label>textarea, -#txt2img_neg_prompt>label>textarea, -#img2img_prompt>label>textarea, -#img2img_neg_prompt>label>textarea, -#control_prompt>label>textarea, -#control_neg_prompt>label>textarea { - font-size: 1.0em; - line-height: 1.4em; - border-radius: var(--radius-md); -} - #txt2img_styles, #img2img_styles, #control_styles { @@ -746,11 +725,6 @@ svg.feather.feather-image, margin-left: 1em; } -#settings_search textarea { - padding: 0.5em; - height: 2.2em !important; -} - #txt2img_cfg_scale { min-width: 200px; } @@ -762,12 +736,6 @@ svg.feather.feather-image, margin-bottom: 0.2em; } -textarea[rows="1"] { - height: 33px !important; - width: 99% !important; - padding: 8px !important; -} - #extras_upscale { margin-top: 10px; } @@ -1054,6 +1022,10 @@ textarea[rows="1"] { padding: 0px; } +#text2img_prompt label, #text2img_neg_prompt label { + margin: 0px; +} + /* Based on Gradio Built-in Dark Theme */ :root, .light, From 88c8a42971ba60da4136fd6280cd503f66c7c02f Mon Sep 17 00:00:00 2001 From: P-Hellmann Date: Fri, 29 Nov 2024 10:58:40 +0100 Subject: [PATCH 010/249] Rearrange forms and tab-nav --- javascript/black-teal-reimagined.css | 34 ++++++++++++---------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/javascript/black-teal-reimagined.css b/javascript/black-teal-reimagined.css index 70315782d..9e7d357a4 100644 --- a/javascript/black-teal-reimagined.css +++ b/javascript/black-teal-reimagined.css @@ -228,16 +228,17 @@ input[type='range']::-moz-range-track { .tab-nav { display: flex; /* Use flexbox for layout */ - justify-content: space-around; + justify-content: space-evenly; /* Space out the tabs evenly */ align-items: center; /* Center items vertically */ background: var(--background-color); /* Background color */ - border-bottom: 1px solid var(--highlight-color) !important; + border-bottom: 3px solid var(--highlight-color) !important; /* Bottom border for separation */ box-shadow: var(--shadow-md); /* Shadow for depth */ + margin-bottom: 5px; } /* Individual Tab Styles */ @@ -246,19 +247,24 @@ input[type='range']::-moz-range-track { /* No background for default state */ color: var(--text-color); /* Text color */ - border: none; + border: 1px solid var(--highlight-color); /* No border */ - border-radius: var(--radius-xxxl); + border-radius: var(--radius-xxl); /* Rounded corners */ cursor: pointer; /* Pointer cursor */ transition: background 0.3s ease, color 0.3s ease; /* Smooth transition */ + padding-top: 5px; + padding-bottom: 5px; + padding-right: 10px; + padding-left: 10px; + margin-bottom: 3px; } /* Active Tab Style */ -.tab-nav>button.active { - background: var(--highlight-color); +.tab-nav>button.selected { + background: var(--primary-100); /* Highlight active tab */ color: var(--background-color); /* Change text color for active tab */ @@ -386,7 +392,8 @@ div.form { border-width: 0; box-shadow: var(--shadow-md); background: var(--background-fill-primary); - padding: 16px; + border-bottom: 3px solid var(--highlight-color); + padding: 3px; border-radius: var(--radius-md); } @@ -720,11 +727,6 @@ svg.feather.feather-image, width: 15em; } -#settings_search { - margin-top: 1em; - margin-left: 1em; -} - #txt2img_cfg_scale { min-width: 200px; } @@ -749,7 +751,6 @@ svg.feather.feather-image, min-width: var(--left-column); max-width: var(--left-column); background-color: var(--neutral-950); - padding-top: 16px; } #pnginfo_html2_info { @@ -837,11 +838,6 @@ svg.feather.feather-image, background: var(--highlight-color); } -/* Extra Networks Tab */ -.extra-networks-tab { - padding: 0 !important; -} - /* Subdirectories Styles */ .extra-network-subdirs { background: var(--input-background-fill); @@ -1014,7 +1010,7 @@ svg.feather.feather-image, display: flex; flex-direction: column; justify-content: space-evenly; - padding: 10px; + padding: 5px; } #txt2img_prompt_container { From eeb595021fd4defc15fd218493b9259feb11d85a Mon Sep 17 00:00:00 2001 From: P-Hellmann Date: Fri, 29 Nov 2024 11:23:07 +0100 Subject: [PATCH 011/249] small changes --- extensions-builtin/sdnext-modernui | 2 +- javascript/black-teal-reimagined.css | 44 ++++------------------------ 2 files changed, 7 insertions(+), 39 deletions(-) diff --git a/extensions-builtin/sdnext-modernui b/extensions-builtin/sdnext-modernui index 3008cee4b..f083ce41a 160000 --- a/extensions-builtin/sdnext-modernui +++ b/extensions-builtin/sdnext-modernui @@ -1 +1 @@ -Subproject commit 3008cee4b67bb00f8f1a4fe4510ec27ba92aa418 +Subproject commit f083ce41a9f18b500f26745ea9e86855e509d2cb diff --git a/javascript/black-teal-reimagined.css b/javascript/black-teal-reimagined.css index 9e7d357a4..eb6942b8c 100644 --- a/javascript/black-teal-reimagined.css +++ b/javascript/black-teal-reimagined.css @@ -216,6 +216,7 @@ input[type='range']::-moz-range-track { ::-webkit-scrollbar-track { background: var(--scrollbar-bg); + border-radius: var(--radius-lg); } ::-webkit-scrollbar-thumb { @@ -234,11 +235,14 @@ input[type='range']::-moz-range-track { /* Center items vertically */ background: var(--background-color); /* Background color */ - border-bottom: 3px solid var(--highlight-color) !important; + border-bottom: 1px dashed var(--highlight-color) !important; /* Bottom border for separation */ box-shadow: var(--shadow-md); /* Shadow for depth */ margin-bottom: 5px; + /* Add some space between the tab nav and the content */ + padding-bottom: 5px; + /* Add space between buttons and border */ } /* Individual Tab Styles */ @@ -395,6 +399,7 @@ div.form { border-bottom: 3px solid var(--highlight-color); padding: 3px; border-radius: var(--radius-md); + margin: 1px; } /* Gradio Style Classes */ @@ -772,12 +777,6 @@ svg.feather.feather-image, /* Extra Networks Container */ #extra_networks_root { - width: 300px; - /* Set a fixed width for the sidebar */ - position: absolute; - height: auto; - right: 0; - top: 13em; z-index: 100; background: var(--background-color); box-shadow: var(--shadow-md); @@ -797,15 +796,6 @@ svg.feather.feather-image, /* Extra Networks Styles */ .extra-networks { background: var(--background-color); - padding: var(--block-label-padding); - border-radius: var(--radius-lg); -} - -/* Extra Networks Div Styles */ -.extra-networks>div { - margin: 0; - border-bottom: none !important; - gap: 0.3em 0; } .extra-networks .tab-nav>button:hover { @@ -822,32 +812,10 @@ svg.feather.feather-image, margin-top: 50px; } -/* Individual Buttons */ -.extra-networks .buttons>button { - margin-left: -0.2em; - height: 1.4em; - color: var(--primary-300) !important; - font-size: 20px !important; - background: var(--button-primary-background-fill); - border: none; - border-radius: var(--radius-sm); - transition: var(--transition); -} - .extra-networks .buttons>button:hover { background: var(--highlight-color); } -/* Subdirectories Styles */ -.extra-network-subdirs { - background: var(--input-background-fill); - overflow-x: hidden; - overflow-y: auto; - min-width: 120px; - padding-top: 0.5em; - margin-top: -4px !important; -} - /* Extra Networks Page */ .extra-networks-page { display: flex; From d4d00aab5945b145f75010fcf93394a7d84c67c2 Mon Sep 17 00:00:00 2001 From: P-Hellmann Date: Fri, 29 Nov 2024 12:14:24 +0100 Subject: [PATCH 012/249] networks page rework --- javascript/black-teal-reimagined.css | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/javascript/black-teal-reimagined.css b/javascript/black-teal-reimagined.css index eb6942b8c..b1e840348 100644 --- a/javascript/black-teal-reimagined.css +++ b/javascript/black-teal-reimagined.css @@ -816,11 +816,6 @@ svg.feather.feather-image, background: var(--highlight-color); } -/* Extra Networks Page */ -.extra-networks-page { - display: flex; -} - /* Network Cards Container */ .extra-network-cards { display: flex; @@ -828,6 +823,8 @@ svg.feather.feather-image, overflow-y: auto; overflow-x: hidden; align-content: flex-start; + padding-top: 20px; + justify-content: center; width: 100%; /* Ensures it takes full width */ } @@ -872,16 +869,23 @@ svg.feather.feather-image, box-shadow: var(--button-shadow); min-height: 30px; border-radius: var(--radius-md); + z-index: 9999; } /* Hover Effects */ +.extra-network-cards .card:hover { + transform: scale(1.3); + z-index: 9999; /* Use a high value to ensure it appears on top */ + transition: transform 0.3s ease, z-index 0s; /* Smooth transition */ +} + .extra-network-cards .card:hover .overlay { - background: rgba(0, 0, 0, 0.70); + z-index: 10000; /* Ensure overlay is also on top */ } .extra-network-cards .card:hover .preview { box-shadow: none; - filter: grayscale(100%); + filter: grayscale(0%); } /* Tags Styles */ @@ -913,6 +917,15 @@ svg.feather.feather-image, font-size: 34px !important; } +.extra-network-cards .card .actions { + background: none; +} + +.extra-network-cards .card .actions .details { + bottom: 50px; + background-color: var(--neutral-800); +} + .extra-network-cards .card .actions>span:hover { color: var(--highlight-color); } From f4bf6271c83f9d748a86cab747b94823959e6322 Mon Sep 17 00:00:00 2001 From: P-Hellmann Date: Fri, 29 Nov 2024 12:24:19 +0100 Subject: [PATCH 013/249] mini changes --- javascript/black-teal-reimagined.css | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/javascript/black-teal-reimagined.css b/javascript/black-teal-reimagined.css index b1e840348..b7567ce75 100644 --- a/javascript/black-teal-reimagined.css +++ b/javascript/black-teal-reimagined.css @@ -631,11 +631,6 @@ svg.feather.feather-image, color: #888; } -.extra-networks { - border-left: 2px solid var(--highlight-color) !important; - padding-left: 4px; -} - .image-buttons { justify-content: center; gap: 0 !important; @@ -795,7 +790,8 @@ svg.feather.feather-image, /* Extra Networks Styles */ .extra-networks { - background: var(--background-color); + border-left: 2px solid var(--highlight-color) !important; + padding-left: 4px; } .extra-networks .tab-nav>button:hover { From ddc6282c93bfa699191f51215e7eff21c9c33a50 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 29 Nov 2024 07:18:05 -0500 Subject: [PATCH 014/249] detailer add augment setting Signed-off-by: Vladimir Mandic --- modules/devices.py | 2 +- modules/postprocess/yolo.py | 2 +- modules/shared.py | 9 +++++---- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/modules/devices.py b/modules/devices.py index 56ac50091..9ca1863a5 100644 --- a/modules/devices.py +++ b/modules/devices.py @@ -471,7 +471,7 @@ def set_cuda_params(): device_name = get_raw_openvino_device() else: device_name = torch.device(get_optimal_device_name()) - log.info(f'Torch parameters: backend={backend} device={device_name} config={opts.cuda_dtype} dtype={dtype} vae={dtype_vae} unet={dtype_unet} context={inference_context.__name__} nohalf={opts.no_half} nohalfvae={opts.no_half_vae} upscast={opts.upcast_sampling} deterministic={opts.cudnn_deterministic} test-fp16={fp16_ok} test-bf16={bf16_ok} optimization="{opts.cross_attention_optimization}"') + log.info(f'Torch parameters: backend={backend} device={device_name} config={opts.cuda_dtype} dtype={dtype} vae={dtype_vae} unet={dtype_unet} context={inference_context.__name__} nohalf={opts.no_half} nohalfvae={opts.no_half_vae} upcast={opts.upcast_sampling} deterministic={opts.cudnn_deterministic} test-fp16={fp16_ok} test-bf16={bf16_ok} optimization="{opts.cross_attention_optimization}"') def cond_cast_unet(tensor): diff --git a/modules/postprocess/yolo.py b/modules/postprocess/yolo.py index f42b6bb9f..5deab1282 100644 --- a/modules/postprocess/yolo.py +++ b/modules/postprocess/yolo.py @@ -72,7 +72,7 @@ def predict( imgsz: int = 640, half: bool = True, device = devices.device, - augment: bool = True, + augment: bool = shared.opts.detailer_augment, agnostic: bool = False, retina: bool = False, mask: bool = True, diff --git a/modules/shared.py b/modules/shared.py index 7dead4dba..bc8def057 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -824,7 +824,7 @@ def get_default_modes(): 'postprocessing_enable_in_main_ui': OptionInfo([], "Additional postprocessing operations", gr.Dropdown, lambda: {"multiselect":True, "choices": [x.name for x in shared_items.postprocessing_scripts()]}), 'postprocessing_operation_order': OptionInfo([], "Postprocessing operation order", gr.Dropdown, lambda: {"multiselect":True, "choices": [x.name for x in shared_items.postprocessing_scripts()]}), - "postprocessing_sep_img2img": OptionInfo("

Img2Img & Inpainting

", "", gr.HTML), + "postprocessing_sep_img2img": OptionInfo("

Inpaint

", "", gr.HTML), "img2img_color_correction": OptionInfo(False, "Apply color correction"), "mask_apply_overlay": OptionInfo(True, "Apply mask as overlay"), "img2img_background_color": OptionInfo("#ffffff", "Image transparent color fill", gr.ColorPicker, {}), @@ -832,7 +832,7 @@ def get_default_modes(): "initial_noise_multiplier": OptionInfo(1.0, "Noise multiplier for image processing", gr.Slider, {"minimum": 0.1, "maximum": 1.5, "step": 0.01, "visible": not native}), "img2img_extra_noise": OptionInfo(0.0, "Extra noise multiplier for img2img", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01, "visible": not native}), - # "postprocessing_sep_detailer": OptionInfo("

Detailer

", "", gr.HTML), + "postprocessing_sep_detailer": OptionInfo("

Detailer

", "", gr.HTML), "detailer_model": OptionInfo("Detailer", "Detailer model", gr.Radio, lambda: {"choices": [x.name() for x in detailers], "visible": False}), "detailer_classes": OptionInfo("", "Detailer classes", gr.Textbox, { "visible": False}), "detailer_conf": OptionInfo(0.6, "Min confidence", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.05, "visible": False}), @@ -844,11 +844,12 @@ def get_default_modes(): "detailer_blur": OptionInfo(10, "Item edge blur", gr.Slider, {"minimum": 0, "maximum": 100, "step": 1, "visible": False}), "detailer_strength": OptionInfo(0.5, "Detailer strength", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01, "visible": False}), "detailer_models": OptionInfo(['face-yolo8n'], "Detailer models", gr.Dropdown, lambda: {"multiselect":True, "choices": list(yolo.list), "visible": False}), - "code_former_weight": OptionInfo(0.2, "CodeFormer weight parameter", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01, "visible": False}), "detailer_unload": OptionInfo(False, "Move detailer model to CPU when complete"), + "detailer_augment": OptionInfo(True, "Detailer use model augment"), "postprocessing_sep_face_restore": OptionInfo("

Face restore

", "", gr.HTML), - "face_restoration_model": OptionInfo("Face restorer", "Face restoration", gr.Radio, lambda: {"choices": ['None'] + [x.name() for x in face_restorers]}), + "face_restoration_model": OptionInfo("None", "Face restoration", gr.Radio, lambda: {"choices": ['None'] + [x.name() for x in face_restorers]}), + "code_former_weight": OptionInfo(0.2, "CodeFormer weight parameter", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01}), "postprocessing_sep_upscalers": OptionInfo("

Upscaling

", "", gr.HTML), "upscaler_unload": OptionInfo(False, "Unload upscaler after processing"), From fabf74d94edde43deeffb41998aad3d846ee9cd0 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 29 Nov 2024 07:49:25 -0500 Subject: [PATCH 015/249] update modernui reference Signed-off-by: Vladimir Mandic --- extensions-builtin/sdnext-modernui | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions-builtin/sdnext-modernui b/extensions-builtin/sdnext-modernui index f083ce41a..3008cee4b 160000 --- a/extensions-builtin/sdnext-modernui +++ b/extensions-builtin/sdnext-modernui @@ -1 +1 @@ -Subproject commit f083ce41a9f18b500f26745ea9e86855e509d2cb +Subproject commit 3008cee4b67bb00f8f1a4fe4510ec27ba92aa418 From 91ca683d548f37b93d70e88d75d0e0ac115a3a62 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 29 Nov 2024 08:01:26 -0500 Subject: [PATCH 016/249] lint fixes Signed-off-by: Vladimir Mandic --- extensions-builtin/sdnext-modernui | 2 +- modules/lora/lora.py | 8 -------- modules/lora/lora_convert.py | 17 +++++++++-------- modules/lora/network.py | 6 ++++-- modules/lora/network_norm.py | 1 + modules/lora/network_oft.py | 3 ++- modules/lora/networks.py | 14 +++++++++----- 7 files changed, 26 insertions(+), 25 deletions(-) delete mode 100644 modules/lora/lora.py diff --git a/extensions-builtin/sdnext-modernui b/extensions-builtin/sdnext-modernui index 3008cee4b..f083ce41a 160000 --- a/extensions-builtin/sdnext-modernui +++ b/extensions-builtin/sdnext-modernui @@ -1 +1 @@ -Subproject commit 3008cee4b67bb00f8f1a4fe4510ec27ba92aa418 +Subproject commit f083ce41a9f18b500f26745ea9e86855e509d2cb diff --git a/modules/lora/lora.py b/modules/lora/lora.py deleted file mode 100644 index 33adfe05c..000000000 --- a/modules/lora/lora.py +++ /dev/null @@ -1,8 +0,0 @@ -# import networks -# -# list_available_loras = networks.list_available_networks -# available_loras = networks.available_networks -# available_lora_aliases = networks.available_network_aliases -# available_lora_hash_lookup = networks.available_network_hash_lookup -# forbidden_lora_aliases = networks.forbidden_network_aliases -# loaded_loras = networks.loaded_networks diff --git a/modules/lora/lora_convert.py b/modules/lora/lora_convert.py index 6bf563125..dc86a24cf 100644 --- a/modules/lora/lora_convert.py +++ b/modules/lora/lora_convert.py @@ -107,14 +107,14 @@ def make_unet_conversion_map() -> Dict[str, str]: class KeyConvert: def __init__(self): - self.is_sdxl = True if shared.sd_model_type == "sdxl" else False - self.UNET_CONVERSION_MAP = make_unet_conversion_map() if self.is_sdxl else None - self.LORA_PREFIX_UNET = "lora_unet_" - self.LORA_PREFIX_TEXT_ENCODER = "lora_te_" - self.OFT_PREFIX_UNET = "oft_unet_" - # SDXL: must starts with LORA_PREFIX_TEXT_ENCODER - self.LORA_PREFIX_TEXT_ENCODER1 = "lora_te1_" - self.LORA_PREFIX_TEXT_ENCODER2 = "lora_te2_" + self.is_sdxl = True if shared.sd_model_type == "sdxl" else False + self.UNET_CONVERSION_MAP = make_unet_conversion_map() if self.is_sdxl else None + self.LORA_PREFIX_UNET = "lora_unet_" + self.LORA_PREFIX_TEXT_ENCODER = "lora_te_" + self.OFT_PREFIX_UNET = "oft_unet_" + # SDXL: must starts with LORA_PREFIX_TEXT_ENCODER + self.LORA_PREFIX_TEXT_ENCODER1 = "lora_te1_" + self.LORA_PREFIX_TEXT_ENCODER2 = "lora_te2_" def __call__(self, key): if self.is_sdxl: @@ -446,6 +446,7 @@ def _convert_sd_scripts_to_ai_toolkit(sds_sd): lora_name_alpha = f"{lora_name}.alpha" diffusers_name = _convert_text_encoder_lora_key(key, lora_name) + sd_lora_rank = 1 if lora_name.startswith(("lora_te_", "lora_te1_")): down_weight = sds_sd.pop(key) sd_lora_rank = down_weight.shape[0] diff --git a/modules/lora/network.py b/modules/lora/network.py index 0785ef9f4..8e6f87368 100644 --- a/modules/lora/network.py +++ b/modules/lora/network.py @@ -1,9 +1,11 @@ import os -from collections import namedtuple import enum +from typing import Union +from collections import namedtuple from modules import sd_models, hashes, shared + NetworkWeights = namedtuple('NetworkWeights', ['network_key', 'sd_key', 'w', 'sd_module']) metadata_tags_order = {"ss_sd_model_name": 1, "ss_resolution": 2, "ss_clip_skip": 3, "ss_num_train_images": 10, "ss_tag_frequency": 20} @@ -105,7 +107,7 @@ def __init__(self, name, network_on_disk: NetworkOnDisk): class ModuleType: - def create_module(self, net: Network, weights: NetworkWeights) -> Network | None: # pylint: disable=W0613 + def create_module(self, net: Network, weights: NetworkWeights) -> Union[Network, None]: # pylint: disable=W0613 return None diff --git a/modules/lora/network_norm.py b/modules/lora/network_norm.py index e8f1740e3..5d059e92e 100644 --- a/modules/lora/network_norm.py +++ b/modules/lora/network_norm.py @@ -1,5 +1,6 @@ import modules.lora.network as network + class ModuleTypeNorm(network.ModuleType): def create_module(self, net: network.Network, weights: network.NetworkWeights): if all(x in weights.w for x in ["w_norm", "b_norm"]): diff --git a/modules/lora/network_oft.py b/modules/lora/network_oft.py index 808286066..e2e61ad45 100644 --- a/modules/lora/network_oft.py +++ b/modules/lora/network_oft.py @@ -1,7 +1,7 @@ import torch +from einops import rearrange import modules.lora.network as network from modules.lora.lyco_helpers import factorization -from einops import rearrange class ModuleTypeOFT(network.ModuleType): @@ -10,6 +10,7 @@ def create_module(self, net: network.Network, weights: network.NetworkWeights): return NetworkModuleOFT(net, weights) return None + # Supports both kohya-ss' implementation of COFT https://github.com/kohya-ss/sd-scripts/blob/main/networks/oft.py # and KohakuBlueleaf's implementation of OFT/COFT https://github.com/KohakuBlueleaf/LyCORIS/blob/dev/lycoris/modules/diag_oft.py class NetworkModuleOFT(network.NetworkModule): # pylint: disable=abstract-method diff --git a/modules/lora/networks.py b/modules/lora/networks.py index c6fde3e04..737623b1e 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -3,6 +3,9 @@ import re import time import concurrent +import torch +import diffusers.models.lora + import modules.lora.network as network import modules.lora.network_lora as network_lora import modules.lora.network_hada as network_hada @@ -14,8 +17,6 @@ import modules.lora.network_glora as network_glora import modules.lora.network_overrides as network_overrides import modules.lora.lora_convert as lora_convert -import torch -import diffusers.models.lora from modules import shared, devices, sd_models, sd_models_compile, errors, scripts, files_cache, model_quant @@ -74,7 +75,7 @@ def assign_network_names_to_compvis_modules(sd_model): shared.sd_model.network_layer_mapping = network_layer_mapping -def load_diffusers(name, network_on_disk, lora_scale=shared.opts.extra_networks_default_multiplier) -> network.Network | None: +def load_diffusers(name, network_on_disk, lora_scale=shared.opts.extra_networks_default_multiplier) -> Union[network.Network, None]: name = name.replace(".", "_") shared.log.debug(f'Load network: type=LoRA name="{name}" file="{network_on_disk.filename}" detected={network_on_disk.sd_version} method=diffusers scale={lora_scale} fuse={shared.opts.lora_fuse_diffusers}') if not shared.native: @@ -103,7 +104,7 @@ def load_diffusers(name, network_on_disk, lora_scale=shared.opts.extra_networks_ return net -def load_network(name, network_on_disk) -> network.Network | None: +def load_network(name, network_on_disk) -> Union[network.Network, None]: if not shared.sd_loaded: return None @@ -173,6 +174,7 @@ def load_network(name, network_on_disk) -> network.Network | None: net.bundle_embeddings = bundle_embeddings return net + def maybe_recompile_model(names, te_multipliers): recompile_model = False if shared.compiled_model_state is not None and shared.compiled_model_state.is_compiled: @@ -186,7 +188,7 @@ def maybe_recompile_model(names, te_multipliers): if not recompile_model: if len(loaded_networks) > 0 and debug: shared.log.debug('Model Compile: Skipping LoRa loading') - return + return recompile_model else: recompile_model = True shared.compiled_model_state.lora_model = [] @@ -277,6 +279,7 @@ def load_networks(names, te_multipliers=None, unet_multipliers=None, dyn_dims=No t1 = time.time() timer['load'] += t1 - t0 + def set_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv], updown, ex_bias): weights_backup = getattr(self, "network_weights_backup", None) bias_backup = getattr(self, "network_bias_backup", None) @@ -389,6 +392,7 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn t1 = time.time() timer['apply'] += t1 - t0 + def network_load(): sd_model = getattr(shared.sd_model, "pipe", shared.sd_model) # wrapped model compatiblility for component_name in ['text_encoder','text_encoder_2', 'unet', 'transformer']: From ed6d47535aaba47ae2862d0ca745dd7102ab7646 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 29 Nov 2024 09:39:38 -0500 Subject: [PATCH 017/249] modules.lora full integration Signed-off-by: Vladimir Mandic --- .../Lora/scripts/lora_script.py | 1 + modules/api/api.py | 2 + modules/api/endpoints.py | 10 +++ modules/extensions.py | 2 +- modules/extra_networks.py | 7 ++- modules/hashes.py | 7 +++ modules/infotext.py | 35 +++++++++++ modules/loader.py | 2 +- modules/lora/networks.py | 25 -------- modules/sd_checkpoint.py | 6 ++ modules/shared.py | 5 +- modules/ui_extra_networks.py | 10 +-- modules/{lora => }/ui_extra_networks_lora.py | 0 modules/ui_models.py | 6 +- scripts/lora_script.py | 62 ------------------- webui.py | 19 +++++- 16 files changed, 98 insertions(+), 101 deletions(-) rename modules/{lora => }/ui_extra_networks_lora.py (100%) delete mode 100644 scripts/lora_script.py diff --git a/extensions-builtin/Lora/scripts/lora_script.py b/extensions-builtin/Lora/scripts/lora_script.py index dea2985b3..24723dd7f 100644 --- a/extensions-builtin/Lora/scripts/lora_script.py +++ b/extensions-builtin/Lora/scripts/lora_script.py @@ -56,6 +56,7 @@ def network_replacement(m): hashes = {x[0].strip().replace(",", ""): x[1].strip() for x in hashes} d["Prompt"] = re.sub(re_lora, network_replacement, d["Prompt"]) + if not shared.native: script_callbacks.on_app_started(api_networks) script_callbacks.on_before_ui(before_ui) diff --git a/modules/api/api.py b/modules/api/api.py index d48cbf521..7d2c2f279 100644 --- a/modules/api/api.py +++ b/modules/api/api.py @@ -79,6 +79,7 @@ def __init__(self, app: FastAPI, queue_lock: Lock): self.add_api_route("/sdapi/v1/sd-vae", endpoints.get_sd_vaes, methods=["GET"], response_model=List[models.ItemVae]) self.add_api_route("/sdapi/v1/extensions", endpoints.get_extensions_list, methods=["GET"], response_model=List[models.ItemExtension]) self.add_api_route("/sdapi/v1/extra-networks", endpoints.get_extra_networks, methods=["GET"], response_model=List[models.ItemExtraNetwork]) + self.add_api_route("/sdapi/v1/loras", endpoints.get_loras, methods=["GET"], response_model=List[dict]) # functional api self.add_api_route("/sdapi/v1/png-info", endpoints.post_pnginfo, methods=["POST"], response_model=models.ResImageInfo) @@ -88,6 +89,7 @@ def __init__(self, app: FastAPI, queue_lock: Lock): self.add_api_route("/sdapi/v1/unload-checkpoint", endpoints.post_unload_checkpoint, methods=["POST"]) self.add_api_route("/sdapi/v1/reload-checkpoint", endpoints.post_reload_checkpoint, methods=["POST"]) self.add_api_route("/sdapi/v1/refresh-vae", endpoints.post_refresh_vae, methods=["POST"]) + self.add_api_route("/sdapi/v1/refresh-loras", endpoints.post_refresh_loras, methods=["POST"]) self.add_api_route("/sdapi/v1/history", endpoints.get_history, methods=["GET"], response_model=List[str]) self.add_api_route("/sdapi/v1/history", endpoints.post_history, methods=["POST"], response_model=int) diff --git a/modules/api/endpoints.py b/modules/api/endpoints.py index 61993db84..1c56b7171 100644 --- a/modules/api/endpoints.py +++ b/modules/api/endpoints.py @@ -40,6 +40,12 @@ def convert_embeddings(embeddings): return {"loaded": convert_embeddings(db.word_embeddings), "skipped": convert_embeddings(db.skipped_embeddings)} +def get_loras(): + from modules.lora import network, networks + def create_lora_json(obj: network.NetworkOnDisk): + return { "name": obj.name, "alias": obj.alias, "path": obj.filename, "metadata": obj.metadata } + return [create_lora_json(obj) for obj in networks.available_networks.values()] + def get_extra_networks(page: Optional[str] = None, name: Optional[str] = None, filename: Optional[str] = None, title: Optional[str] = None, fullname: Optional[str] = None, hash: Optional[str] = None): # pylint: disable=redefined-builtin res = [] for pg in shared.extra_networks: @@ -126,6 +132,10 @@ def post_refresh_checkpoints(): def post_refresh_vae(): return shared.refresh_vaes() +def post_refresh_loras(): + from modules.lora import networks + return networks.list_available_networks() + def get_extensions_list(): from modules import extensions extensions.list_extensions() diff --git a/modules/extensions.py b/modules/extensions.py index 5a8a53d29..ccd92dbf0 100644 --- a/modules/extensions.py +++ b/modules/extensions.py @@ -154,4 +154,4 @@ def list_extensions(): for dirname, path, is_builtin in extension_paths: extension = Extension(name=dirname, path=path, enabled=dirname not in disabled_extensions, is_builtin=is_builtin) extensions.append(extension) - shared.log.info(f'Disabled extensions: {[e.name for e in extensions if not e.enabled]}') + shared.log.debug(f'Disabled extensions: {[e.name for e in extensions if not e.enabled]}') diff --git a/modules/extra_networks.py b/modules/extra_networks.py index b464bd349..010157af9 100644 --- a/modules/extra_networks.py +++ b/modules/extra_networks.py @@ -15,10 +15,13 @@ def register_extra_network(extra_network): def register_default_extra_networks(): - from modules.ui_extra_networks_hypernet import ExtraNetworkHypernet - register_extra_network(ExtraNetworkHypernet()) from modules.ui_extra_networks_styles import ExtraNetworkStyles register_extra_network(ExtraNetworkStyles()) + from modules.lora.extra_networks_lora import ExtraNetworkLora + register_extra_network(ExtraNetworkLora()) + if shared.opts.hypernetwork_enabled: + from modules.ui_extra_networks_hypernet import ExtraNetworkHypernet + register_extra_network(ExtraNetworkHypernet()) class ExtraNetworkParams: diff --git a/modules/hashes.py b/modules/hashes.py index cf83794b0..a003f4840 100644 --- a/modules/hashes.py +++ b/modules/hashes.py @@ -9,6 +9,13 @@ cache_data = None progress_ok = True + +def init_cache(): + global cache_data # pylint: disable=global-statement + if cache_data is None: + cache_data = {} if not os.path.isfile(cache_filename) else shared.readfile(cache_filename, lock=True) + + def dump_cache(): shared.writefile(cache_data, cache_filename) diff --git a/modules/infotext.py b/modules/infotext.py index 05e06e600..4b9dd15ff 100644 --- a/modules/infotext.py +++ b/modules/infotext.py @@ -10,6 +10,7 @@ debug = lambda *args, **kwargs: None # pylint: disable=unnecessary-lambda-assignment re_size = re.compile(r"^(\d+)x(\d+)$") # int x int re_param = re.compile(r'\s*([\w ]+):\s*("(?:\\"[^,]|\\"|\\|[^\"])+"|[^,]*)(?:,|$)') # multi-word: value +re_lora = re.compile("") - if added: - params["Prompt"] += "\n" + "".join(added) - - -list_available_networks() diff --git a/modules/sd_checkpoint.py b/modules/sd_checkpoint.py index e035fc3db..a95ade0b1 100644 --- a/modules/sd_checkpoint.py +++ b/modules/sd_checkpoint.py @@ -275,6 +275,12 @@ def select_checkpoint(op='model'): return checkpoint_info +def init_metadata(): + global sd_metadata # pylint: disable=global-statement + if sd_metadata is None: + sd_metadata = shared.readfile(sd_metadata_file, lock=True) if os.path.isfile(sd_metadata_file) else {} + + def read_metadata_from_safetensors(filename): global sd_metadata # pylint: disable=global-statement if sd_metadata is None: diff --git a/modules/shared.py b/modules/shared.py index bc8def057..9e00cd165 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -132,7 +132,8 @@ def readfile(filename, silent=False, lock=False): # data = json.loads(data) t1 = time.time() if not silent: - log.debug(f'Read: file="{filename}" json={len(data)} bytes={os.path.getsize(filename)} time={t1-t0:.3f}') + fn = f'{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access + log.debug(f'Read: file="{filename}" json={len(data)} bytes={os.path.getsize(filename)} time={t1-t0:.3f} fn={fn}') except FileNotFoundError as err: log.debug(f'Reading failed: {filename} {err}') except Exception as err: @@ -363,7 +364,7 @@ def list_samplers(): def temp_disable_extensions(): disable_safe = ['sd-webui-controlnet', 'multidiffusion-upscaler-for-automatic1111', 'a1111-sd-webui-lycoris', 'sd-webui-agent-scheduler', 'clip-interrogator-ext', 'stable-diffusion-webui-rembg', 'sd-extension-chainner', 'stable-diffusion-webui-images-browser'] - disable_diffusers = ['sd-webui-controlnet', 'multidiffusion-upscaler-for-automatic1111', 'a1111-sd-webui-lycoris', 'sd-webui-animatediff'] + disable_diffusers = ['sd-webui-controlnet', 'multidiffusion-upscaler-for-automatic1111', 'a1111-sd-webui-lycoris', 'sd-webui-animatediff', 'Lora'] disable_themes = ['sd-webui-lobe-theme', 'cozy-nest', 'sdnext-modernui'] disable_original = [] disabled = [] diff --git a/modules/ui_extra_networks.py b/modules/ui_extra_networks.py index 1365e1c24..3b803076a 100644 --- a/modules/ui_extra_networks.py +++ b/modules/ui_extra_networks.py @@ -460,17 +460,19 @@ def register_page(page: ExtraNetworksPage): def register_pages(): - from modules.ui_extra_networks_textual_inversion import ExtraNetworksPageTextualInversion from modules.ui_extra_networks_checkpoints import ExtraNetworksPageCheckpoints - from modules.ui_extra_networks_styles import ExtraNetworksPageStyles + from modules.ui_extra_networks_lora import ExtraNetworksPageLora from modules.ui_extra_networks_vae import ExtraNetworksPageVAEs + from modules.ui_extra_networks_styles import ExtraNetworksPageStyles from modules.ui_extra_networks_history import ExtraNetworksPageHistory + from modules.ui_extra_networks_textual_inversion import ExtraNetworksPageTextualInversion debug('EN register-pages') register_page(ExtraNetworksPageCheckpoints()) - register_page(ExtraNetworksPageStyles()) - register_page(ExtraNetworksPageTextualInversion()) + register_page(ExtraNetworksPageLora()) register_page(ExtraNetworksPageVAEs()) + register_page(ExtraNetworksPageStyles()) register_page(ExtraNetworksPageHistory()) + register_page(ExtraNetworksPageTextualInversion()) if shared.opts.hypernetwork_enabled: from modules.ui_extra_networks_hypernets import ExtraNetworksPageHypernetworks register_page(ExtraNetworksPageHypernetworks()) diff --git a/modules/lora/ui_extra_networks_lora.py b/modules/ui_extra_networks_lora.py similarity index 100% rename from modules/lora/ui_extra_networks_lora.py rename to modules/ui_extra_networks_lora.py diff --git a/modules/ui_models.py b/modules/ui_models.py index 624c3849d..7ab8b0d07 100644 --- a/modules/ui_models.py +++ b/modules/ui_models.py @@ -8,7 +8,7 @@ from modules.ui_components import ToolButton from modules.ui_common import create_refresh_button from modules.call_queue import wrap_gradio_gpu_call -from modules.shared import opts, log, req, readfile, max_workers +from modules.shared import opts, log, req, readfile, max_workers, native import modules.ui_symbols import modules.errors import modules.hashes @@ -794,6 +794,10 @@ def civit_update_download(): civit_results4.select(fn=civit_update_select, inputs=[civit_results4], outputs=[models_outcome, civit_update_download_btn]) civit_update_download_btn.click(fn=civit_update_download, inputs=[], outputs=[models_outcome]) + if native: + from modules.lora.lora_extract import create_ui as lora_extract_ui + lora_extract_ui() + for ui in extra_ui: if callable(ui): ui() diff --git a/scripts/lora_script.py b/scripts/lora_script.py deleted file mode 100644 index a153a2caa..000000000 --- a/scripts/lora_script.py +++ /dev/null @@ -1,62 +0,0 @@ -import re -import modules.lora.networks as networks -from modules.lora.lora_extract import create_ui -from modules.lora.network import NetworkOnDisk -from modules.lora.ui_extra_networks_lora import ExtraNetworksPageLora -from modules.lora.extra_networks_lora import ExtraNetworkLora -from modules import script_callbacks, extra_networks, ui_extra_networks, ui_models, shared # pylint: disable=unused-import - - -re_lora = re.compile(" Date: Fri, 29 Nov 2024 10:05:06 -0500 Subject: [PATCH 018/249] conditional imports and summary timer Signed-off-by: Vladimir Mandic --- modules/api/api.py | 7 +++++-- modules/infotext.py | 1 + modules/lora/networks.py | 8 +++++++- modules/processing_callbacks.py | 5 ++++- modules/processing_diffusers.py | 11 ++++++----- modules/ui_extra_networks.py | 7 ++++--- webui.py | 7 ++++--- 7 files changed, 31 insertions(+), 15 deletions(-) diff --git a/modules/api/api.py b/modules/api/api.py index 7d2c2f279..b958085ea 100644 --- a/modules/api/api.py +++ b/modules/api/api.py @@ -79,7 +79,6 @@ def __init__(self, app: FastAPI, queue_lock: Lock): self.add_api_route("/sdapi/v1/sd-vae", endpoints.get_sd_vaes, methods=["GET"], response_model=List[models.ItemVae]) self.add_api_route("/sdapi/v1/extensions", endpoints.get_extensions_list, methods=["GET"], response_model=List[models.ItemExtension]) self.add_api_route("/sdapi/v1/extra-networks", endpoints.get_extra_networks, methods=["GET"], response_model=List[models.ItemExtraNetwork]) - self.add_api_route("/sdapi/v1/loras", endpoints.get_loras, methods=["GET"], response_model=List[dict]) # functional api self.add_api_route("/sdapi/v1/png-info", endpoints.post_pnginfo, methods=["POST"], response_model=models.ResImageInfo) @@ -89,10 +88,14 @@ def __init__(self, app: FastAPI, queue_lock: Lock): self.add_api_route("/sdapi/v1/unload-checkpoint", endpoints.post_unload_checkpoint, methods=["POST"]) self.add_api_route("/sdapi/v1/reload-checkpoint", endpoints.post_reload_checkpoint, methods=["POST"]) self.add_api_route("/sdapi/v1/refresh-vae", endpoints.post_refresh_vae, methods=["POST"]) - self.add_api_route("/sdapi/v1/refresh-loras", endpoints.post_refresh_loras, methods=["POST"]) self.add_api_route("/sdapi/v1/history", endpoints.get_history, methods=["GET"], response_model=List[str]) self.add_api_route("/sdapi/v1/history", endpoints.post_history, methods=["POST"], response_model=int) + # lora api + if shared.native: + self.add_api_route("/sdapi/v1/loras", endpoints.get_loras, methods=["GET"], response_model=List[dict]) + self.add_api_route("/sdapi/v1/refresh-loras", endpoints.post_refresh_loras, methods=["POST"]) + # gallery api gallery.register_api(app) diff --git a/modules/infotext.py b/modules/infotext.py index 4b9dd15ff..baa995c88 100644 --- a/modules/infotext.py +++ b/modules/infotext.py @@ -28,6 +28,7 @@ def unquote(text): return text +# disabled by default can be enabled if needed def check_lora(params): try: import modules.lora.networks as networks diff --git a/modules/lora/networks.py b/modules/lora/networks.py index dc6d86b2f..2db145a5a 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -17,7 +17,7 @@ import modules.lora.network_glora as network_glora import modules.lora.network_overrides as network_overrides import modules.lora.lora_convert as lora_convert -from modules import shared, devices, sd_models, sd_models_compile, errors, scripts, files_cache, model_quant +from modules import shared, devices, sd_models, sd_models_compile, errors, files_cache, model_quant debug = os.environ.get('SD_LORA_DEBUG', None) is not None @@ -44,6 +44,10 @@ ] +def total_time(): + return sum(timer.values()) + + def assign_network_names_to_compvis_modules(sd_model): if sd_model is None: return @@ -394,6 +398,8 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn def network_load(): + for k in timer.keys(): + timer[k] = 0 sd_model = getattr(shared.sd_model, "pipe", shared.sd_model) # wrapped model compatiblility for component_name in ['text_encoder','text_encoder_2', 'unet', 'transformer']: component = getattr(sd_model, component_name, None) diff --git a/modules/processing_callbacks.py b/modules/processing_callbacks.py index 0bb94abad..e1bf723cc 100644 --- a/modules/processing_callbacks.py +++ b/modules/processing_callbacks.py @@ -6,6 +6,7 @@ from modules import shared, processing_correction, extra_networks, timer, prompt_parser_diffusers from modules.lora.networks import network_load + p = None debug = os.environ.get('SD_CALLBACK_DEBUG', None) is not None debug_callback = shared.log.trace if debug else lambda *args, **kwargs: None @@ -15,6 +16,7 @@ def set_callbacks_p(processing): global p # pylint: disable=global-statement p = processing + def prompt_callback(step, kwargs): if prompt_parser_diffusers.embedder is None or 'prompt_embeds' not in kwargs: return kwargs @@ -29,6 +31,7 @@ def prompt_callback(step, kwargs): debug_callback(f"Callback: {e}") return kwargs + def diffusers_callback_legacy(step: int, timestep: int, latents: typing.Union[torch.FloatTensor, np.ndarray]): if p is None: return @@ -64,7 +67,7 @@ def diffusers_callback(pipe, step: int = 0, timestep: int = 0, kwargs: dict = {} if shared.state.interrupted or shared.state.skipped: raise AssertionError('Interrupted...') time.sleep(0.1) - if hasattr(p, "stepwise_lora"): + if hasattr(p, "stepwise_lora") and shared.native: extra_networks.activate(p, p.extra_network_data, step=step) network_load() if latents is None: diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 2e8fb357c..ae24f5f80 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -8,8 +8,7 @@ from modules.processing_helpers import resize_hires, calculate_base_steps, calculate_hires_steps, calculate_refiner_steps, save_intermediate, update_sampler, is_txt2img, is_refiner_enabled from modules.processing_args import set_pipeline_args from modules.onnx_impl import preprocess_pipeline as preprocess_onnx_pipeline, check_parameters_changed as olive_check_parameters_changed -from modules.lora.networks import network_load -from modules.lora.networks import timer as network_timer +from modules.lora import networks debug = shared.log.trace if os.environ.get('SD_DIFFUSERS_DEBUG', None) is not None else lambda *args, **kwargs: None @@ -427,9 +426,9 @@ def process_diffusers(p: processing.StableDiffusionProcessing): p.prompts = p.all_prompts[p.iteration * p.batch_size:(p.iteration+1) * p.batch_size] if p.negative_prompts is None or len(p.negative_prompts) == 0: p.negative_prompts = p.all_negative_prompts[p.iteration * p.batch_size:(p.iteration+1) * p.batch_size] - network_timer['apply'] = 0 - network_timer['restore'] = 0 - network_load() + + # load loras + networks.network_load() sd_models.move_model(shared.sd_model, devices.device) sd_models_compile.openvino_recompile_model(p, hires=False, refiner=False) # recompile if a parameter changes @@ -459,6 +458,8 @@ def process_diffusers(p: processing.StableDiffusionProcessing): results = process_decode(p, output) timer.process.record('decode') + timer.process.add('lora', networks.total_time()) + shared.sd_model = orig_pipeline if p.state == '': global last_p # pylint: disable=global-statement diff --git a/modules/ui_extra_networks.py b/modules/ui_extra_networks.py index 3b803076a..89cea44b8 100644 --- a/modules/ui_extra_networks.py +++ b/modules/ui_extra_networks.py @@ -460,19 +460,20 @@ def register_page(page: ExtraNetworksPage): def register_pages(): + debug('EN register-pages') from modules.ui_extra_networks_checkpoints import ExtraNetworksPageCheckpoints - from modules.ui_extra_networks_lora import ExtraNetworksPageLora from modules.ui_extra_networks_vae import ExtraNetworksPageVAEs from modules.ui_extra_networks_styles import ExtraNetworksPageStyles from modules.ui_extra_networks_history import ExtraNetworksPageHistory from modules.ui_extra_networks_textual_inversion import ExtraNetworksPageTextualInversion - debug('EN register-pages') register_page(ExtraNetworksPageCheckpoints()) - register_page(ExtraNetworksPageLora()) register_page(ExtraNetworksPageVAEs()) register_page(ExtraNetworksPageStyles()) register_page(ExtraNetworksPageHistory()) register_page(ExtraNetworksPageTextualInversion()) + if shared.native: + from modules.ui_extra_networks_lora import ExtraNetworksPageLora + register_page(ExtraNetworksPageLora()) if shared.opts.hypernetwork_enabled: from modules.ui_extra_networks_hypernets import ExtraNetworksPageHypernetworks register_page(ExtraNetworksPageHypernetworks()) diff --git a/webui.py b/webui.py index 3aae34447..2b8d7c56f 100644 --- a/webui.py +++ b/webui.py @@ -34,7 +34,6 @@ import modules.upscaler import modules.textual_inversion.textual_inversion import modules.hypernetworks.hypernetwork -import modules.lora.networks import modules.script_callbacks from modules.api.middleware import setup_middleware from modules.shared import cmd_opts, opts # pylint: disable=unused-import @@ -104,8 +103,10 @@ def initialize(): modules.sd_models.setup_model() timer.startup.record("models") - modules.lora.networks.list_available_networks() - timer.startup.record("lora") + if shared.native: + import modules.lora.networks as lora_networks + lora_networks.list_available_networks() + timer.startup.record("lora") shared.prompt_styles.reload() timer.startup.record("styles") From 998f75c14d0f0fca7c9d619f85db673a43fac43c Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 29 Nov 2024 10:22:47 -0500 Subject: [PATCH 019/249] cleanup Signed-off-by: Vladimir Mandic --- modules/modelloader.py | 4 ++-- modules/sd_checkpoint.py | 6 +++++- modules/ui_extra_networks_lora.py | 2 +- webui.py | 6 +++--- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/modules/modelloader.py b/modules/modelloader.py index b1b3930d6..b022b4fc6 100644 --- a/modules/modelloader.py +++ b/modules/modelloader.py @@ -267,7 +267,7 @@ def download_diffusers_model(hub_id: str, cache_dir: str = None, download_config def load_diffusers_models(clear=True): excluded_models = [] - t0 = time.time() + # t0 = time.time() place = shared.opts.diffusers_dir if place is None or len(place) == 0 or not os.path.isdir(place): place = os.path.join(models_path, 'Diffusers') @@ -316,7 +316,7 @@ def load_diffusers_models(clear=True): debug(f'Error analyzing diffusers model: "{folder}" {e}') except Exception as e: shared.log.error(f"Error listing diffusers: {place} {e}") - shared.log.debug(f'Scanning diffusers cache: folder="{place}" items={len(list(diffuser_repos))} time={time.time()-t0:.2f}') + # shared.log.debug(f'Scanning diffusers cache: folder="{place}" items={len(list(diffuser_repos))} time={time.time()-t0:.2f}') return diffuser_repos diff --git a/modules/sd_checkpoint.py b/modules/sd_checkpoint.py index a95ade0b1..2f6533ef0 100644 --- a/modules/sd_checkpoint.py +++ b/modules/sd_checkpoint.py @@ -123,13 +123,17 @@ def list_models(): checkpoint_aliases.clear() ext_filter = [".safetensors"] if shared.opts.sd_disable_ckpt or shared.native else [".ckpt", ".safetensors"] model_list = list(modelloader.load_models(model_path=model_path, model_url=None, command_path=shared.opts.ckpt_dir, ext_filter=ext_filter, download_name=None, ext_blacklist=[".vae.ckpt", ".vae.safetensors"])) + safetensors_list = [] for filename in sorted(model_list, key=str.lower): checkpoint_info = CheckpointInfo(filename) + safetensors_list.append(checkpoint_info) if checkpoint_info.name is not None: checkpoint_info.register() + diffusers_list = [] if shared.native: for repo in modelloader.load_diffusers_models(clear=True): checkpoint_info = CheckpointInfo(repo['name'], sha=repo['hash']) + diffusers_list.append(checkpoint_info) if checkpoint_info.name is not None: checkpoint_info.register() if shared.cmd_opts.ckpt is not None: @@ -143,7 +147,7 @@ def list_models(): shared.opts.data['sd_model_checkpoint'] = checkpoint_info.title elif shared.cmd_opts.ckpt != shared.default_sd_model_file and shared.cmd_opts.ckpt is not None: shared.log.warning(f'Load model: path="{shared.cmd_opts.ckpt}" not found') - shared.log.info(f'Available Models: path="{shared.opts.ckpt_dir}" items={len(checkpoints_list)} time={time.time()-t0:.2f}') + shared.log.info(f'Available Models: items={len(checkpoints_list)} safetensors="{shared.opts.ckpt_dir}":{len(safetensors_list)} diffusers="{shared.opts.diffusers_dir}":{len(diffusers_list)} time={time.time()-t0:.2f}') checkpoints_list = dict(sorted(checkpoints_list.items(), key=lambda cp: cp[1].filename)) def update_model_hashes(): diff --git a/modules/ui_extra_networks_lora.py b/modules/ui_extra_networks_lora.py index 73cce47a3..9dd1b3573 100644 --- a/modules/ui_extra_networks_lora.py +++ b/modules/ui_extra_networks_lora.py @@ -120,4 +120,4 @@ def list_items(self): return items def allowed_directories_for_previews(self): - return [shared.cmd_opts.lora_dir, shared.cmd_opts.lyco_dir] + return [shared.cmd_opts.lora_dir] diff --git a/webui.py b/webui.py index 2b8d7c56f..4eb6e89ce 100644 --- a/webui.py +++ b/webui.py @@ -96,9 +96,6 @@ def initialize(): modules.model_te.refresh_te_list() timer.startup.record("te") - extensions.list_extensions() - timer.startup.record("extensions") - modelloader.cleanup_models() modules.sd_models.setup_model() timer.startup.record("models") @@ -120,6 +117,9 @@ def initialize(): yolo.initialize() timer.startup.record("detailer") + extensions.list_extensions() + timer.startup.record("extensions") + log.info('Load extensions') t_timer, t_total = modules.scripts.load_scripts() timer.startup.record("extensions") From 6187696d1d1d0349e5e23927f83673681b6f2cd4 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 29 Nov 2024 22:35:00 +0300 Subject: [PATCH 020/249] Fix sequential offload with lora --- modules/lora/networks.py | 8 +++++--- modules/sd_models.py | 14 +++++++++----- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/modules/lora/networks.py b/modules/lora/networks.py index 2db145a5a..8a23f7413 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -278,8 +278,6 @@ def load_networks(names, te_multipliers=None, unet_multipliers=None, dyn_dims=No shared.sd_model = sd_models_compile.compile_diffusers(shared.sd_model) shared.compiled_model_state.lora_model = backup_lora_model - if shared.opts.diffusers_offload_mode == "balanced": - sd_models.apply_balanced_offload(shared.sd_model) t1 = time.time() timer['load'] += t1 - t0 @@ -401,12 +399,16 @@ def network_load(): for k in timer.keys(): timer[k] = 0 sd_model = getattr(shared.sd_model, "pipe", shared.sd_model) # wrapped model compatiblility + if shared.opts.diffusers_offload_mode == "sequential": + sd_models.disable_offload(sd_model) + sd_models.move_model(sd_model, device=devices.cpu) for component_name in ['text_encoder','text_encoder_2', 'unet', 'transformer']: component = getattr(sd_model, component_name, None) if component is not None: for _, module in component.named_modules(): network_apply_weights(module) - + if shared.opts.diffusers_offload_mode == "sequential": + sd_models.set_diffuser_offload(sd_model, op="model") def list_available_networks(): t0 = time.time() diff --git a/modules/sd_models.py b/modules/sd_models.py index 68446bdd3..361f6375b 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -405,7 +405,7 @@ def apply_balanced_offload_to_module(pipe): if hasattr(pipe, "_internal_dict"): keys = pipe._internal_dict.keys() # pylint: disable=protected-access else: - keys = get_signature(shared.sd_model).keys() + keys = get_signature(pipe).keys() for module_name in keys: # pylint: disable=protected-access module = getattr(pipe, module_name, None) if isinstance(module, torch.nn.Module): @@ -1448,10 +1448,14 @@ def disable_offload(sd_model): from accelerate.hooks import remove_hook_from_module if not getattr(sd_model, 'has_accelerate', False): return - if hasattr(sd_model, 'components'): - for _name, model in sd_model.components.items(): - if isinstance(model, torch.nn.Module): - remove_hook_from_module(model, recurse=True) + if hasattr(sd_model, "_internal_dict"): + keys = sd_model._internal_dict.keys() # pylint: disable=protected-access + else: + keys = get_signature(sd_model).keys() + for module_name in keys: # pylint: disable=protected-access + module = getattr(sd_model, module_name, None) + if isinstance(module, torch.nn.Module): + module = remove_hook_from_module(module, recurse=True) sd_model.has_accelerate = False From 76d17080f82ee9a0f3146b7aa4ad9fe87f159788 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 29 Nov 2024 22:53:17 +0300 Subject: [PATCH 021/249] revert networs.py --- modules/lora/networks.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/modules/lora/networks.py b/modules/lora/networks.py index 8a23f7413..2db145a5a 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -278,6 +278,8 @@ def load_networks(names, te_multipliers=None, unet_multipliers=None, dyn_dims=No shared.sd_model = sd_models_compile.compile_diffusers(shared.sd_model) shared.compiled_model_state.lora_model = backup_lora_model + if shared.opts.diffusers_offload_mode == "balanced": + sd_models.apply_balanced_offload(shared.sd_model) t1 = time.time() timer['load'] += t1 - t0 @@ -399,16 +401,12 @@ def network_load(): for k in timer.keys(): timer[k] = 0 sd_model = getattr(shared.sd_model, "pipe", shared.sd_model) # wrapped model compatiblility - if shared.opts.diffusers_offload_mode == "sequential": - sd_models.disable_offload(sd_model) - sd_models.move_model(sd_model, device=devices.cpu) for component_name in ['text_encoder','text_encoder_2', 'unet', 'transformer']: component = getattr(sd_model, component_name, None) if component is not None: for _, module in component.named_modules(): network_apply_weights(module) - if shared.opts.diffusers_offload_mode == "sequential": - sd_models.set_diffuser_offload(sd_model, op="model") + def list_available_networks(): t0 = time.time() From 10dc19812ebac02b8dda87efe232452344906e85 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 29 Nov 2024 22:55:15 +0300 Subject: [PATCH 022/249] revert sd_models.py --- modules/sd_models.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/modules/sd_models.py b/modules/sd_models.py index 361f6375b..68446bdd3 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -405,7 +405,7 @@ def apply_balanced_offload_to_module(pipe): if hasattr(pipe, "_internal_dict"): keys = pipe._internal_dict.keys() # pylint: disable=protected-access else: - keys = get_signature(pipe).keys() + keys = get_signature(shared.sd_model).keys() for module_name in keys: # pylint: disable=protected-access module = getattr(pipe, module_name, None) if isinstance(module, torch.nn.Module): @@ -1448,14 +1448,10 @@ def disable_offload(sd_model): from accelerate.hooks import remove_hook_from_module if not getattr(sd_model, 'has_accelerate', False): return - if hasattr(sd_model, "_internal_dict"): - keys = sd_model._internal_dict.keys() # pylint: disable=protected-access - else: - keys = get_signature(sd_model).keys() - for module_name in keys: # pylint: disable=protected-access - module = getattr(sd_model, module_name, None) - if isinstance(module, torch.nn.Module): - module = remove_hook_from_module(module, recurse=True) + if hasattr(sd_model, 'components'): + for _name, model in sd_model.components.items(): + if isinstance(model, torch.nn.Module): + remove_hook_from_module(model, recurse=True) sd_model.has_accelerate = False From a969c195b1140aab97fb935af7ad7829f245b9ae Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 29 Nov 2024 15:40:09 -0500 Subject: [PATCH 023/249] lora stats Signed-off-by: Vladimir Mandic --- installer.py | 2 +- scripts/flux_tools.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/installer.py b/installer.py index 37202552d..8fb6d9683 100644 --- a/installer.py +++ b/installer.py @@ -459,7 +459,7 @@ def check_python(supported_minors=[9, 10, 11, 12], reason=None): def check_diffusers(): if args.skip_all or args.skip_requirements: return - sha = '069186fac510d6f6f88a5e435523b235c823a8a0' + sha = 'c96bfa5c80eca798d555a79a491043c311d0f608' pkg = pkg_resources.working_set.by_key.get('diffusers', None) minor = int(pkg.version.split('.')[1] if pkg is not None else 0) cur = opts.get('diffusers_version', '') if minor > 0 else '' diff --git a/scripts/flux_tools.py b/scripts/flux_tools.py index e5fe443b7..3fbab6c6f 100644 --- a/scripts/flux_tools.py +++ b/scripts/flux_tools.py @@ -100,7 +100,7 @@ def run(self, p: processing.StableDiffusionProcessing, tool: str = 'None', stren if tool == 'Depth': # pipe = FluxControlPipeline.from_pretrained("black-forest-labs/FLUX.1-Depth-dev", torch_dtype=torch.bfloat16, revision="refs/pr/1").to("cuda") - install('git+https://github.com/asomoza/image_gen_aux.git', 'image_gen_aux') + install('git+https://github.com/huggingface/image_gen_aux.git', 'image_gen_aux') if shared.sd_model.__class__.__name__ != 'FluxControlPipeline' or 'Depth' not in shared.opts.sd_model_checkpoint: shared.opts.data["sd_model_checkpoint"] = "black-forest-labs/FLUX.1-Depth-dev" sd_models.reload_model_weights(op='model', revision="refs/pr/1") From 4a141ba196edf38ab83e72e0118ed91b008d4c94 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 29 Nov 2024 15:40:20 -0500 Subject: [PATCH 024/249] lora stats Signed-off-by: Vladimir Mandic --- modules/lora/networks.py | 69 +++++++++++++++++++++++++++------------- modules/model_flux.py | 8 ++--- 2 files changed, 51 insertions(+), 26 deletions(-) diff --git a/modules/lora/networks.py b/modules/lora/networks.py index 2db145a5a..beb4634c2 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -5,6 +5,7 @@ import concurrent import torch import diffusers.models.lora +import rich.progress as p import modules.lora.network as network import modules.lora.network_lora as network_lora @@ -21,11 +22,12 @@ debug = os.environ.get('SD_LORA_DEBUG', None) is not None +pbar = p.Progress(p.TextColumn('[cyan]LoRA apply'), p.BarColumn(), p.TaskProgressColumn(), p.TimeRemainingColumn(), p.TimeElapsedColumn(), p.TextColumn('[cyan]{task.description}'), console=shared.console) extra_network_lora = None available_networks = {} available_network_aliases = {} loaded_networks: List[network.Network] = [] -timer = { 'load': 0, 'apply': 0, 'restore': 0, 'deactivate': 0 } +timer = { 'list': 0, 'load': 0, 'backup': 0, 'calc': 0, 'apply': 0, 'restore': 0, 'deactivate': 0 } lora_cache = {} diffuser_loaded = [] diffuser_scales = [] @@ -216,7 +218,6 @@ def load_networks(names, te_multipliers=None, unet_multipliers=None, dyn_dims=No loaded_networks.clear() diffuser_loaded.clear() diffuser_scales.clear() - timer['load'] = 0 t0 = time.time() for i, (network_on_disk, name) in enumerate(zip(networks_on_disk, names)): @@ -269,8 +270,6 @@ def load_networks(names, te_multipliers=None, unet_multipliers=None, dyn_dims=No if len(loaded_networks) > 0 and debug: shared.log.debug(f'Load network: type=LoRA loaded={len(loaded_networks)} cache={list(lora_cache)}') - devices.torch_gc() - if recompile_model: shared.log.info("Load network: type=LoRA recompiling model") backup_lora_model = shared.compiled_model_state.lora_model @@ -278,13 +277,18 @@ def load_networks(names, te_multipliers=None, unet_multipliers=None, dyn_dims=No shared.sd_model = sd_models_compile.compile_diffusers(shared.sd_model) shared.compiled_model_state.lora_model = backup_lora_model - if shared.opts.diffusers_offload_mode == "balanced": - sd_models.apply_balanced_offload(shared.sd_model) + + if len(loaded_networks) > 0: + devices.torch_gc() + if shared.opts.diffusers_offload_mode == "balanced": + sd_models.apply_balanced_offload(shared.sd_model) + t1 = time.time() - timer['load'] += t1 - t0 + timer['load'] = t1 - t0 def set_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv], updown, ex_bias): + t0 = time.time() weights_backup = getattr(self, "network_weights_backup", None) bias_backup = getattr(self, "network_bias_backup", None) if weights_backup is None and bias_backup is None: @@ -315,9 +319,12 @@ def set_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm else: self.bias = None self.to(device) + t1 = time.time() + timer['apply'] += t1 - t0 def maybe_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv], wanted_names): # pylint: disable=W0613 + t0 = time.time() weights_backup = getattr(self, "network_weights_backup", None) if weights_backup is None and wanted_names != (): # pylint: disable=C1803 if getattr(self.weight, "quant_type", None) in ['nf4', 'fp4']: @@ -344,6 +351,8 @@ def maybe_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn. if shared.opts.lora_offload_backup and bias_backup is not None: bias_backup = bias_backup.to(devices.cpu) self.network_bias_backup = bias_backup + t1 = time.time() + timer['backup'] += t1 - t0 def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv]): @@ -353,16 +362,13 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn If not, restores orginal weights from backup and alters weights according to networks. """ network_layer_name = getattr(self, 'network_layer_name', None) - if network_layer_name is None: - return - t0 = time.time() current_names = getattr(self, "network_current_names", ()) wanted_names = tuple((x.name, x.te_multiplier, x.unet_multiplier, x.dyn_dim) for x in loaded_networks) - if any([net.modules.get(network_layer_name, None) for net in loaded_networks]): # noqa: C419 # pylint: disable=R1729 - maybe_backup_weights(self, wanted_names) + maybe_backup_weights(self, wanted_names) if current_names != wanted_names: batch_updown = None batch_ex_bias = None + t0 = time.time() for net in loaded_networks: # default workflow where module is known and has weights module = net.modules.get(network_layer_name, None) @@ -391,21 +397,39 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn continue shared.log.warning(f'LoRA network="{net.name}" layer="{network_layer_name}" unsupported operation') extra_network_lora.errors[net.name] = extra_network_lora.errors.get(net.name, 0) + 1 + t1 = time.time() + timer['calc'] += t1 - t0 set_weights(self, batch_updown, batch_ex_bias) # Set or restore weights from backup self.network_current_names = wanted_names - t1 = time.time() - timer['apply'] += t1 - t0 -def network_load(): - for k in timer.keys(): - timer[k] = 0 +def network_load(): # called from processing + timer['backup'] = 0 + timer['calc'] = 0 + timer['apply'] = 0 sd_model = getattr(shared.sd_model, "pipe", shared.sd_model) # wrapped model compatiblility - for component_name in ['text_encoder','text_encoder_2', 'unet', 'transformer']: - component = getattr(sd_model, component_name, None) - if component is not None: - for _, module in component.named_modules(): - network_apply_weights(module) + with pbar: + for component_name in ['text_encoder','text_encoder_2', 'unet', 'transformer']: + component = getattr(sd_model, component_name, None) + if component is not None: + applied = 0 + modules = list(component.named_modules()) + task_start = time.time() + task = pbar.add_task(description=component_name , total=len(modules), visible=False) + for _, module in modules: + layer_name = getattr(module, 'network_layer_name', None) + if layer_name is None: + continue + present = any([net.modules.get(layer_name, None) for net in loaded_networks]) # noqa: C419 + if present: + network_apply_weights(module) + applied += 1 + pbar.update(task, advance=1, visible=(time.time() - task_start) > 1) # progress bar becomes visible if operation takes more than 1sec + pbar.remove_task(task) + if debug: + shared.log.debug(f'Load network: type=LoRA component={component_name} modules={len(modules)} applied={applied}') + if debug: + shared.log.debug(f'Load network: type=LoRA total={total_time():.2f} timers={timer}') def list_available_networks(): @@ -442,4 +466,5 @@ def add_network(filename): for fn in candidates: executor.submit(add_network, fn) t1 = time.time() + timer['list'] = t1 - t0 shared.log.info(f'Available LoRAs: path="{shared.cmd_opts.lora_dir}" items={len(available_networks)} folders={len(forbidden_network_aliases)} time={t1 - t0:.2f}') diff --git a/modules/model_flux.py b/modules/model_flux.py index 324e50b36..ce2c55f70 100644 --- a/modules/model_flux.py +++ b/modules/model_flux.py @@ -223,10 +223,8 @@ def load_flux(checkpoint_info, diffusers_load_config): # triggered by opts.sd_ch if shared.opts.sd_unet != 'None': try: debug(f'Load model: type=FLUX unet="{shared.opts.sd_unet}"') - _transformer = load_transformer(sd_unet.unet_dict[shared.opts.sd_unet]) - if _transformer is not None: - transformer = _transformer - else: + transformer = load_transformer(sd_unet.unet_dict[shared.opts.sd_unet]) + if transformer is None: shared.opts.sd_unet = 'None' sd_unet.failed_unet.append(shared.opts.sd_unet) except Exception as e: @@ -334,6 +332,8 @@ def load_flux(checkpoint_info, diffusers_load_config): # triggered by opts.sd_ch text_encoder_1 = None text_encoder_2 = None vae = None + for k in kwargs.keys(): + kwargs[k] = None devices.torch_gc() return pipe From 9624cdfc9327794aa8a88b18985ba6768271ab6d Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sat, 30 Nov 2024 00:16:38 +0300 Subject: [PATCH 025/249] Fix offload issues with lora --- modules/lora/networks.py | 14 +++++++++++--- modules/sd_models.py | 14 +++++++++----- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/modules/lora/networks.py b/modules/lora/networks.py index beb4634c2..f211149bd 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -280,8 +280,6 @@ def load_networks(names, te_multipliers=None, unet_multipliers=None, dyn_dims=No if len(loaded_networks) > 0: devices.torch_gc() - if shared.opts.diffusers_offload_mode == "balanced": - sd_models.apply_balanced_offload(shared.sd_model) t1 = time.time() timer['load'] = t1 - t0 @@ -375,7 +373,7 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn if module is not None and hasattr(self, 'weight'): try: with devices.inference_context(): - weight = self.weight # calculate quant weights once + weight = self.weight.to(devices.device) # calculate quant weights once updown, ex_bias = module.calc_updown(weight) if batch_updown is not None and updown is not None: batch_updown += updown @@ -385,6 +383,11 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn batch_ex_bias += ex_bias else: batch_ex_bias = ex_bias + if shared.opts.diffusers_offload_mode != "none": + if batch_updown is not None: + batch_updown = batch_updown.to(devices.cpu) + if batch_ex_bias is not None: + batch_ex_bias = batch_ex_bias.to(devices.cpu) except RuntimeError as e: extra_network_lora.errors[net.name] = extra_network_lora.errors.get(net.name, 0) + 1 if debug: @@ -408,6 +411,9 @@ def network_load(): # called from processing timer['calc'] = 0 timer['apply'] = 0 sd_model = getattr(shared.sd_model, "pipe", shared.sd_model) # wrapped model compatiblility + if shared.opts.diffusers_offload_mode != "none": + sd_models.disable_offload(sd_model) + sd_models.move_model(sd_model, device=devices.cpu) with pbar: for component_name in ['text_encoder','text_encoder_2', 'unet', 'transformer']: component = getattr(sd_model, component_name, None) @@ -428,6 +434,8 @@ def network_load(): # called from processing pbar.remove_task(task) if debug: shared.log.debug(f'Load network: type=LoRA component={component_name} modules={len(modules)} applied={applied}') + if shared.opts.diffusers_offload_mode != "none": + sd_models.set_diffuser_offload(sd_model, op="model") if debug: shared.log.debug(f'Load network: type=LoRA total={total_time():.2f} timers={timer}') diff --git a/modules/sd_models.py b/modules/sd_models.py index 68446bdd3..361f6375b 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -405,7 +405,7 @@ def apply_balanced_offload_to_module(pipe): if hasattr(pipe, "_internal_dict"): keys = pipe._internal_dict.keys() # pylint: disable=protected-access else: - keys = get_signature(shared.sd_model).keys() + keys = get_signature(pipe).keys() for module_name in keys: # pylint: disable=protected-access module = getattr(pipe, module_name, None) if isinstance(module, torch.nn.Module): @@ -1448,10 +1448,14 @@ def disable_offload(sd_model): from accelerate.hooks import remove_hook_from_module if not getattr(sd_model, 'has_accelerate', False): return - if hasattr(sd_model, 'components'): - for _name, model in sd_model.components.items(): - if isinstance(model, torch.nn.Module): - remove_hook_from_module(model, recurse=True) + if hasattr(sd_model, "_internal_dict"): + keys = sd_model._internal_dict.keys() # pylint: disable=protected-access + else: + keys = get_signature(sd_model).keys() + for module_name in keys: # pylint: disable=protected-access + module = getattr(sd_model, module_name, None) + if isinstance(module, torch.nn.Module): + module = remove_hook_from_module(module, recurse=True) sd_model.has_accelerate = False From bb4ba595404befc227352eb5de1642b5ba622f18 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 30 Nov 2024 08:50:25 -0500 Subject: [PATCH 026/249] update lora Signed-off-by: Vladimir Mandic --- launch.py | 3 +++ modules/cmd_args.py | 1 + modules/lora/networks.py | 45 ++++++++++++++++++----------------- modules/processing.py | 9 +++++++ modules/processing_helpers.py | 6 ++++- 5 files changed, 41 insertions(+), 23 deletions(-) diff --git a/launch.py b/launch.py index e00da58c7..5c8a6051a 100755 --- a/launch.py +++ b/launch.py @@ -192,6 +192,9 @@ def main(): global args # pylint: disable=global-statement installer.ensure_base_requirements() init_args() # setup argparser and default folders + if args.malloc: + import tracemalloc + tracemalloc.start() installer.args = args installer.setup_logging() installer.log.info('Starting SD.Next') diff --git a/modules/cmd_args.py b/modules/cmd_args.py index 752ad02c0..cb4e5fc16 100644 --- a/modules/cmd_args.py +++ b/modules/cmd_args.py @@ -26,6 +26,7 @@ def main_args(): group_diag.add_argument("--no-hashing", default=os.environ.get("SD_NOHASHING", False), action='store_true', help="Disable hashing of checkpoints, default: %(default)s") group_diag.add_argument("--no-metadata", default=os.environ.get("SD_NOMETADATA", False), action='store_true', help="Disable reading of metadata from models, default: %(default)s") group_diag.add_argument("--profile", default=os.environ.get("SD_PROFILE", False), action='store_true', help="Run profiler, default: %(default)s") + group_diag.add_argument("--malloc", default=os.environ.get("SD_PROFILE", False), action='store_true', help="Trace memory ops, default: %(default)s") group_diag.add_argument("--disable-queue", default=os.environ.get("SD_DISABLEQUEUE", False), action='store_true', help="Disable queues, default: %(default)s") group_diag.add_argument('--debug', default=os.environ.get("SD_DEBUG", False), action='store_true', help = "Run installer with debug logging, default: %(default)s") diff --git a/modules/lora/networks.py b/modules/lora/networks.py index f211149bd..23c45ff2a 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -22,7 +22,7 @@ debug = os.environ.get('SD_LORA_DEBUG', None) is not None -pbar = p.Progress(p.TextColumn('[cyan]LoRA apply'), p.BarColumn(), p.TaskProgressColumn(), p.TimeRemainingColumn(), p.TimeElapsedColumn(), p.TextColumn('[cyan]{task.description}'), console=shared.console) +pbar = p.Progress(p.TextColumn('[cyan]{task.description}'), p.BarColumn(), p.TaskProgressColumn(), p.TimeRemainingColumn(), p.TimeElapsedColumn(), console=shared.console) extra_network_lora = None available_networks = {} available_network_aliases = {} @@ -50,6 +50,13 @@ def total_time(): return sum(timer.values()) +def get_timers(): + t = { 'total': round(sum(timer.values()), 2) } + for k, v in timer.items(): + t[k] = round(v, 2) + return t + + def assign_network_names_to_compvis_modules(sd_model): if sd_model is None: return @@ -362,7 +369,8 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn network_layer_name = getattr(self, 'network_layer_name', None) current_names = getattr(self, "network_current_names", ()) wanted_names = tuple((x.name, x.te_multiplier, x.unet_multiplier, x.dyn_dim) for x in loaded_networks) - maybe_backup_weights(self, wanted_names) + if network_layer_name is not None and any([net.modules.get(network_layer_name, None) for net in loaded_networks]): # noqa: C419 + maybe_backup_weights(self, wanted_names) if current_names != wanted_names: batch_updown = None batch_ex_bias = None @@ -414,30 +422,23 @@ def network_load(): # called from processing if shared.opts.diffusers_offload_mode != "none": sd_models.disable_offload(sd_model) sd_models.move_model(sd_model, device=devices.cpu) + modules = [] + for component_name in ['text_encoder','text_encoder_2', 'unet', 'transformer']: + component = getattr(sd_model, component_name, None) + if component is not None and hasattr(component, 'named_modules'): + modules += list(component.named_modules()) with pbar: - for component_name in ['text_encoder','text_encoder_2', 'unet', 'transformer']: - component = getattr(sd_model, component_name, None) - if component is not None: - applied = 0 - modules = list(component.named_modules()) - task_start = time.time() - task = pbar.add_task(description=component_name , total=len(modules), visible=False) - for _, module in modules: - layer_name = getattr(module, 'network_layer_name', None) - if layer_name is None: - continue - present = any([net.modules.get(layer_name, None) for net in loaded_networks]) # noqa: C419 - if present: - network_apply_weights(module) - applied += 1 - pbar.update(task, advance=1, visible=(time.time() - task_start) > 1) # progress bar becomes visible if operation takes more than 1sec - pbar.remove_task(task) - if debug: - shared.log.debug(f'Load network: type=LoRA component={component_name} modules={len(modules)} applied={applied}') + task = pbar.add_task(description='Apply network: type=LoRA' , total=len(modules), visible=len(loaded_networks) > 0) + for _, module in modules: + network_apply_weights(module) + # pbar.update(task, advance=1) # progress bar becomes visible if operation takes more than 1sec + pbar.remove_task(task) + if debug: + shared.log.debug(f'Load network: type=LoRA modules={len(modules)}') if shared.opts.diffusers_offload_mode != "none": sd_models.set_diffuser_offload(sd_model, op="model") if debug: - shared.log.debug(f'Load network: type=LoRA total={total_time():.2f} timers={timer}') + shared.log.debug(f'Load network: type=LoRA timers{get_timers()}') def list_available_networks(): diff --git a/modules/processing.py b/modules/processing.py index 16e7a9213..92faaee8d 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -473,4 +473,13 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: p.scripts.postprocess(p, processed) timer.process.record('post') shared.log.info(f'Processed: images={len(output_images)} its={(p.steps * len(output_images)) / (t1 - t0):.2f} time={t1-t0:.2f} timers={timer.process.dct(min_time=0.02)} memory={memstats.memory_stats()}') + + if shared.cmd_opts.malloc: + import tracemalloc + snapshot = tracemalloc.take_snapshot() + stats = snapshot.statistics('lineno') + shared.log.debug('Profile malloc:') + for stat in stats[:20]: + frame = stat.traceback[0] + shared.log.debug(f' file="{frame.filename}":{frame.lineno} size={stat.size}') return processed diff --git a/modules/processing_helpers.py b/modules/processing_helpers.py index 22acf296c..ab08d4cc8 100644 --- a/modules/processing_helpers.py +++ b/modules/processing_helpers.py @@ -1,4 +1,5 @@ import os +import time import math import random import warnings @@ -9,7 +10,7 @@ from PIL import Image from skimage import exposure from blendmodes.blend import blendLayers, BlendType -from modules import shared, devices, images, sd_models, sd_samplers, sd_hijack_hypertile, processing_vae +from modules import shared, devices, images, sd_models, sd_samplers, sd_hijack_hypertile, processing_vae, timer debug = shared.log.trace if os.environ.get('SD_PROCESS_DEBUG', None) is not None else lambda *args, **kwargs: None @@ -352,6 +353,7 @@ def diffusers_image_conditioning(_source_image, latent_image, _image_mask=None): def validate_sample(tensor): + t0 = time.time() if not isinstance(tensor, np.ndarray) and not isinstance(tensor, torch.Tensor): return tensor dtype = tensor.dtype @@ -377,6 +379,8 @@ def validate_sample(tensor): if upcast is not None and not upcast: setattr(shared.sd_model.vae.config, 'force_upcast', True) # noqa: B010 shared.log.warning('Decode: upcast=True set, retry operation') + t1 = time.time() + timer.process.add('validate', t1 - t0) return cast From 8800cd61b7ab271ffb6dd6cb759fd7fc5c015762 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 30 Nov 2024 09:02:47 -0500 Subject: [PATCH 027/249] add stats Signed-off-by: Vladimir Mandic --- modules/lora/networks.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/modules/lora/networks.py b/modules/lora/networks.py index 23c45ff2a..51ef27a8a 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -28,6 +28,7 @@ available_network_aliases = {} loaded_networks: List[network.Network] = [] timer = { 'list': 0, 'load': 0, 'backup': 0, 'calc': 0, 'apply': 0, 'restore': 0, 'deactivate': 0 } +backup_size = 0 lora_cache = {} diffuser_loaded = [] diffuser_scales = [] @@ -289,6 +290,7 @@ def load_networks(names, te_multipliers=None, unet_multipliers=None, dyn_dims=No devices.torch_gc() t1 = time.time() + backup_size = 0 timer['load'] = t1 - t0 @@ -329,6 +331,7 @@ def set_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm def maybe_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv], wanted_names): # pylint: disable=W0613 + global backup_size # pylint: disable=W0603 t0 = time.time() weights_backup = getattr(self, "network_weights_backup", None) if weights_backup is None and wanted_names != (): # pylint: disable=C1803 @@ -347,6 +350,7 @@ def maybe_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn. if shared.opts.lora_offload_backup and weights_backup is not None: weights_backup = weights_backup.to(devices.cpu) self.network_weights_backup = weights_backup + backup_size += weights_backup.numel() * weights_backup.element_size() bias_backup = getattr(self, "network_bias_backup", None) if bias_backup is None: if getattr(self, 'bias', None) is not None: @@ -356,6 +360,8 @@ def maybe_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn. if shared.opts.lora_offload_backup and bias_backup is not None: bias_backup = bias_backup.to(devices.cpu) self.network_bias_backup = bias_backup + if bias_backup is not None: + backup_size += bias_backup.numel() * bias_backup.element_size() t1 = time.time() timer['backup'] += t1 - t0 @@ -431,14 +437,15 @@ def network_load(): # called from processing task = pbar.add_task(description='Apply network: type=LoRA' , total=len(modules), visible=len(loaded_networks) > 0) for _, module in modules: network_apply_weights(module) - # pbar.update(task, advance=1) # progress bar becomes visible if operation takes more than 1sec + pbar.update(task, advance=1) # progress bar becomes visible if operation takes more than 1sec pbar.remove_task(task) + modules.clear() if debug: shared.log.debug(f'Load network: type=LoRA modules={len(modules)}') if shared.opts.diffusers_offload_mode != "none": sd_models.set_diffuser_offload(sd_model, op="model") if debug: - shared.log.debug(f'Load network: type=LoRA timers{get_timers()}') + shared.log.debug(f'Load network: type=LoRA time={get_timers()} backup={backup_size}') def list_available_networks(): From cebf1a116b025a48dd0cf8a9110a44b300aa515a Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sat, 30 Nov 2024 17:04:35 +0300 Subject: [PATCH 028/249] Disable load lora gpu with medvram too --- modules/sd_models.py | 5 ++++- modules/shared.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/modules/sd_models.py b/modules/sd_models.py index 361f6375b..ccba0bfb5 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -413,11 +413,11 @@ def apply_balanced_offload_to_module(pipe): if checkpoint_name is None: checkpoint_name = pipe.__class__.__name__ offload_dir = os.path.join(shared.opts.accelerate_offload_path, checkpoint_name, module_name) + network_layer_name = getattr(module, "network_layer_name", None) module = remove_hook_from_module(module, recurse=True) try: module = module.to("cpu") module.offload_dir = offload_dir - network_layer_name = getattr(module, "network_layer_name", None) module = add_hook_to_module(module, dispatch_from_cpu_hook(), append=True) module._hf_hook.execution_device = torch.device(devices.device) # pylint: disable=protected-access if network_layer_name: @@ -1455,7 +1455,10 @@ def disable_offload(sd_model): for module_name in keys: # pylint: disable=protected-access module = getattr(sd_model, module_name, None) if isinstance(module, torch.nn.Module): + network_layer_name = getattr(module, "network_layer_name", None) module = remove_hook_from_module(module, recurse=True) + if network_layer_name: + module.network_layer_name = network_layer_name sd_model.has_accelerate = False diff --git a/modules/shared.py b/modules/shared.py index 9e00cd165..ea6a44a3b 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -910,7 +910,7 @@ def get_default_modes(): "lora_apply_tags": OptionInfo(0, "LoRA auto-apply tags", gr.Slider, {"minimum": -1, "maximum": 32, "step": 1}), "lora_in_memory_limit": OptionInfo(0, "LoRA memory cache", gr.Slider, {"minimum": 0, "maximum": 24, "step": 1}), "lora_quant": OptionInfo("NF4","LoRA precision in quantized models", gr.Radio, {"choices": ["NF4", "FP4"]}), - "lora_load_gpu": OptionInfo(True if not cmd_opts.lowvram else False, "Load LoRA directly to GPU"), + "lora_load_gpu": OptionInfo(True if not (cmd_opts.lowvram or cmd_opts.medvram) else False, "Load LoRA directly to GPU"), "lora_offload_backup": OptionInfo(True, "Offload LoRA Backup Weights"), })) From 3a50710b73347160e0a7344508c7af7c9c033ca7 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 30 Nov 2024 12:29:58 -0500 Subject: [PATCH 029/249] lora refactor in progress Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 5 +- modules/lora/extra_networks_lora.py | 9 ++- modules/lora/networks.py | 73 ++++++++++++------------- modules/processing_callbacks.py | 2 - modules/processing_diffusers.py | 21 ++++--- modules/prompt_parser_diffusers.py | 9 ++- modules/sd_models.py | 85 ++++++++++++++++------------- 7 files changed, 109 insertions(+), 95 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 919041bde..dcb88bcf3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Change Log for SD.Next -## Update for 2024-11-28 +## Update for 2024-11-30 ### New models and integrations @@ -67,7 +67,8 @@ - fix xyz-grid with lora - fix api script callbacks - fix gpu memory monitoring -- simplify img2img/inpaint/sketch canvas handling +- simplify img2img/inpaint/sketch canvas handling +- fix prompt caching ## Update for 2024-11-21 diff --git a/modules/lora/extra_networks_lora.py b/modules/lora/extra_networks_lora.py index 3aea659d9..c875ba0d5 100644 --- a/modules/lora/extra_networks_lora.py +++ b/modules/lora/extra_networks_lora.py @@ -113,22 +113,21 @@ def __init__(self): self.errors = {} def activate(self, p, params_list, step=0): - t0 = time.time() self.errors.clear() if self.active: if self.model != shared.opts.sd_model_checkpoint: # reset if model changed self.active = False if len(params_list) > 0 and not self.active: # activate patches once - shared.log.debug(f'Activate network: type=LoRA model="{shared.opts.sd_model_checkpoint}"') + # shared.log.debug(f'Activate network: type=LoRA model="{shared.opts.sd_model_checkpoint}"') self.active = True self.model = shared.opts.sd_model_checkpoint names, te_multipliers, unet_multipliers, dyn_dims = parse(p, params_list, step) - networks.load_networks(names, te_multipliers, unet_multipliers, dyn_dims) - t1 = time.time() + networks.load_networks(names, te_multipliers, unet_multipliers, dyn_dims) # load + networks.network_load() # backup/apply if len(networks.loaded_networks) > 0 and step == 0: infotext(p) prompt(p) - shared.log.info(f'Load network: type=LoRA apply={[n.name for n in networks.loaded_networks]} te={te_multipliers} unet={unet_multipliers} dims={dyn_dims} load={t1-t0:.2f}') + shared.log.info(f'Load network: type=LoRA apply={[n.name for n in networks.loaded_networks]} te={te_multipliers} unet={unet_multipliers} time={networks.get_timers()}') def deactivate(self, p): t0 = time.time() diff --git a/modules/lora/networks.py b/modules/lora/networks.py index 51ef27a8a..86c6e5ed0 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -54,7 +54,8 @@ def total_time(): def get_timers(): t = { 'total': round(sum(timer.values()), 2) } for k, v in timer.items(): - t[k] = round(v, 2) + if v > 0.1: + t[k] = round(v, 2) return t @@ -216,6 +217,7 @@ def maybe_recompile_model(names, te_multipliers): def load_networks(names, te_multipliers=None, unet_multipliers=None, dyn_dims=None): + global backup_size # pylint: disable=global-statement networks_on_disk: list[network.NetworkOnDisk] = [available_network_aliases.get(name, None) for name in names] if any(x is None for x in networks_on_disk): list_available_networks() @@ -304,10 +306,9 @@ def set_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm with devices.inference_context(): if weights_backup is not None: if updown is not None: - if len(weights_backup.shape) == 4 and weights_backup.shape[1] == 9: - # inpainting model. zero pad updown to make channel[1] 4 to 9 + if len(weights_backup.shape) == 4 and weights_backup.shape[1] == 9: # inpainting model. zero pad updown to make channel[1] 4 to 9 updown = torch.nn.functional.pad(updown, (0, 0, 0, 0, 0, 5)) # pylint: disable=not-callable - weights_backup = weights_backup.clone().to(device) + weights_backup = weights_backup.clone().to(self.weight.device) weights_backup += updown.to(weights_backup) if getattr(self, "quant_type", None) in ['nf4', 'fp4']: bnb = model_quant.load_bnb('Load network: type=LoRA', silent=True) @@ -375,18 +376,18 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn network_layer_name = getattr(self, 'network_layer_name', None) current_names = getattr(self, "network_current_names", ()) wanted_names = tuple((x.name, x.te_multiplier, x.unet_multiplier, x.dyn_dim) for x in loaded_networks) - if network_layer_name is not None and any([net.modules.get(network_layer_name, None) for net in loaded_networks]): # noqa: C419 - maybe_backup_weights(self, wanted_names) - if current_names != wanted_names: - batch_updown = None - batch_ex_bias = None - t0 = time.time() - for net in loaded_networks: - # default workflow where module is known and has weights - module = net.modules.get(network_layer_name, None) - if module is not None and hasattr(self, 'weight'): - try: - with devices.inference_context(): + with devices.inference_context(): + if network_layer_name is not None and any([net.modules.get(network_layer_name, None) for net in loaded_networks]): # noqa: C419 + maybe_backup_weights(self, wanted_names) + if current_names != wanted_names: + batch_updown = None + batch_ex_bias = None + t0 = time.time() + for net in loaded_networks: + # default workflow where module is known and has weights + module = net.modules.get(network_layer_name, None) + if module is not None and hasattr(self, 'weight'): + try: weight = self.weight.to(devices.device) # calculate quant weights once updown, ex_bias = module.calc_updown(weight) if batch_updown is not None and updown is not None: @@ -402,22 +403,22 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn batch_updown = batch_updown.to(devices.cpu) if batch_ex_bias is not None: batch_ex_bias = batch_ex_bias.to(devices.cpu) - except RuntimeError as e: - extra_network_lora.errors[net.name] = extra_network_lora.errors.get(net.name, 0) + 1 - if debug: - module_name = net.modules.get(network_layer_name, None) - shared.log.error(f'LoRA apply weight name="{net.name}" module="{module_name}" layer="{network_layer_name}" {e}') - errors.display(e, 'LoRA') - raise RuntimeError('LoRA apply weight') from e - continue - if module is None: - continue - shared.log.warning(f'LoRA network="{net.name}" layer="{network_layer_name}" unsupported operation') - extra_network_lora.errors[net.name] = extra_network_lora.errors.get(net.name, 0) + 1 - t1 = time.time() - timer['calc'] += t1 - t0 - set_weights(self, batch_updown, batch_ex_bias) # Set or restore weights from backup - self.network_current_names = wanted_names + except RuntimeError as e: + extra_network_lora.errors[net.name] = extra_network_lora.errors.get(net.name, 0) + 1 + if debug: + module_name = net.modules.get(network_layer_name, None) + shared.log.error(f'LoRA apply weight name="{net.name}" module="{module_name}" layer="{network_layer_name}" {e}') + errors.display(e, 'LoRA') + raise RuntimeError('LoRA apply weight') from e + continue + if module is None: + continue + shared.log.warning(f'LoRA network="{net.name}" layer="{network_layer_name}" unsupported operation') + extra_network_lora.errors[net.name] = extra_network_lora.errors.get(net.name, 0) + 1 + t1 = time.time() + timer['calc'] += t1 - t0 + set_weights(self, batch_updown, batch_ex_bias) # Set or restore weights from backup + self.network_current_names = wanted_names def network_load(): # called from processing @@ -425,7 +426,7 @@ def network_load(): # called from processing timer['calc'] = 0 timer['apply'] = 0 sd_model = getattr(shared.sd_model, "pipe", shared.sd_model) # wrapped model compatiblility - if shared.opts.diffusers_offload_mode != "none": + if shared.opts.diffusers_offload_mode == "sequential": sd_models.disable_offload(sd_model) sd_models.move_model(sd_model, device=devices.cpu) modules = [] @@ -441,11 +442,9 @@ def network_load(): # called from processing pbar.remove_task(task) modules.clear() if debug: - shared.log.debug(f'Load network: type=LoRA modules={len(modules)}') - if shared.opts.diffusers_offload_mode != "none": + shared.log.debug(f'Load network: type=LoRA modules={len(modules)} backup={backup_size} time={get_timers()}') + if shared.opts.diffusers_offload_mode == "sequential": sd_models.set_diffuser_offload(sd_model, op="model") - if debug: - shared.log.debug(f'Load network: type=LoRA time={get_timers()} backup={backup_size}') def list_available_networks(): diff --git a/modules/processing_callbacks.py b/modules/processing_callbacks.py index e1bf723cc..f3eb0bc37 100644 --- a/modules/processing_callbacks.py +++ b/modules/processing_callbacks.py @@ -4,7 +4,6 @@ import torch import numpy as np from modules import shared, processing_correction, extra_networks, timer, prompt_parser_diffusers -from modules.lora.networks import network_load p = None @@ -69,7 +68,6 @@ def diffusers_callback(pipe, step: int = 0, timestep: int = 0, kwargs: dict = {} time.sleep(0.1) if hasattr(p, "stepwise_lora") and shared.native: extra_networks.activate(p, p.extra_network_data, step=step) - network_load() if latents is None: return kwargs elif shared.opts.nan_skip: diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index ae24f5f80..463a15280 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -199,11 +199,6 @@ def process_hires(p: processing.StableDiffusionProcessing, output): if hasattr(shared.sd_model, "vae") and output.images is not None and len(output.images) > 0: output.images = processing_vae.vae_decode(latents=output.images, model=shared.sd_model, full_quality=p.full_quality, output_type='pil', width=p.hr_upscale_to_x, height=p.hr_upscale_to_y) # controlnet cannnot deal with latent input p.task_args['image'] = output.images # replace so hires uses new output - sd_models.move_model(shared.sd_model, devices.device) - if hasattr(shared.sd_model, 'unet'): - sd_models.move_model(shared.sd_model.unet, devices.device) - if hasattr(shared.sd_model, 'transformer'): - sd_models.move_model(shared.sd_model.transformer, devices.device) update_sampler(p, shared.sd_model, second_pass=True) orig_denoise = p.denoising_strength p.denoising_strength = strength @@ -227,6 +222,11 @@ def process_hires(p: processing.StableDiffusionProcessing, output): shared.state.job = 'HiRes' shared.state.sampling_steps = hires_args.get('prior_num_inference_steps', None) or p.steps or hires_args.get('num_inference_steps', None) try: + sd_models.move_model(shared.sd_model, devices.device) + if hasattr(shared.sd_model, 'unet'): + sd_models.move_model(shared.sd_model.unet, devices.device) + if hasattr(shared.sd_model, 'transformer'): + sd_models.move_model(shared.sd_model.transformer, devices.device) sd_models_compile.check_deepcache(enable=True) output = shared.sd_model(**hires_args) # pylint: disable=not-callable if isinstance(output, dict): @@ -405,6 +405,9 @@ def process_diffusers(p: processing.StableDiffusionProcessing): shared.sd_model = orig_pipeline return results + if shared.opts.diffusers_offload_mode == "balanced": + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) + # sanitize init_images if hasattr(p, 'init_images') and getattr(p, 'init_images', None) is None: del p.init_images @@ -427,10 +430,6 @@ def process_diffusers(p: processing.StableDiffusionProcessing): if p.negative_prompts is None or len(p.negative_prompts) == 0: p.negative_prompts = p.all_negative_prompts[p.iteration * p.batch_size:(p.iteration+1) * p.batch_size] - # load loras - networks.network_load() - - sd_models.move_model(shared.sd_model, devices.device) sd_models_compile.openvino_recompile_model(p, hires=False, refiner=False) # recompile if a parameter changes if 'base' not in p.skip: @@ -461,6 +460,10 @@ def process_diffusers(p: processing.StableDiffusionProcessing): timer.process.add('lora', networks.total_time()) shared.sd_model = orig_pipeline + + if shared.opts.diffusers_offload_mode == "balanced": + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) + if p.state == '': global last_p # pylint: disable=global-statement last_p = p diff --git a/modules/prompt_parser_diffusers.py b/modules/prompt_parser_diffusers.py index 2edef4bf5..c74731c6d 100644 --- a/modules/prompt_parser_diffusers.py +++ b/modules/prompt_parser_diffusers.py @@ -16,6 +16,7 @@ token_dict = None # used by helper get_tokens token_type = None # used by helper get_tokens cache = OrderedDict() +last_attention = None embedder = None @@ -52,7 +53,7 @@ def __init__(self, prompts, negative_prompts, steps, clip_skip, p): self.prompts = prompts self.negative_prompts = negative_prompts self.batchsize = len(self.prompts) - self.attention = None + self.attention = last_attention self.allsame = self.compare_prompts() # collapses batched prompts to single prompt if possible self.steps = steps self.clip_skip = clip_skip @@ -78,6 +79,8 @@ def __init__(self, prompts, negative_prompts, steps, clip_skip, p): self.scheduled_encode(pipe, batchidx) else: self.encode(pipe, prompt, negative_prompt, batchidx) + if shared.opts.diffusers_offload_mode == "balanced": + pipe = sd_models.apply_balanced_offload(pipe) self.checkcache(p) debug(f"Prompt encode: time={(time.time() - t0):.3f}") @@ -113,6 +116,7 @@ def flatten(xss): debug(f"Prompt cache: add={key}") while len(cache) > int(shared.opts.sd_textencoder_cache_size): cache.popitem(last=False) + return True if item: self.__dict__.update(cache[key]) cache.move_to_end(key) @@ -161,7 +165,9 @@ def extend_embeds(self, batchidx, idx): # Extends scheduled prompt via index self.negative_pooleds[batchidx].append(self.negative_pooleds[batchidx][idx]) def encode(self, pipe, positive_prompt, negative_prompt, batchidx): + global last_attention # pylint: disable=global-statement self.attention = shared.opts.prompt_attention + last_attention = self.attention if self.attention == "xhinker": prompt_embed, positive_pooled, negative_embed, negative_pooled = get_xhinker_text_embeddings(pipe, positive_prompt, negative_prompt, self.clip_skip) else: @@ -178,7 +184,6 @@ def encode(self, pipe, positive_prompt, negative_prompt, batchidx): if debug_enabled: get_tokens(pipe, 'positive', positive_prompt) get_tokens(pipe, 'negative', negative_prompt) - pipe = prepare_model() def __call__(self, key, step=0): batch = getattr(self, key) diff --git a/modules/sd_models.py b/modules/sd_models.py index ccba0bfb5..2cf7b3931 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -13,6 +13,7 @@ from rich import progress # pylint: disable=redefined-builtin import torch import safetensors.torch +import accelerate from omegaconf import OmegaConf from ldm.util import instantiate_from_config from modules import paths, shared, shared_state, modelloader, devices, script_callbacks, sd_vae, sd_unet, errors, sd_models_config, sd_models_compile, sd_hijack_accelerate, sd_detect @@ -310,6 +311,7 @@ def set_accelerate(sd_model): def set_diffuser_offload(sd_model, op: str = 'model'): + t0 = time.time() if not shared.native: shared.log.warning('Attempting to use offload with backend=original') return @@ -363,41 +365,50 @@ def set_diffuser_offload(sd_model, op: str = 'model'): sd_model = apply_balanced_offload(sd_model) except Exception as e: shared.log.error(f'Setting {op}: offload={shared.opts.diffusers_offload_mode} {e}') + process_timer.add('offload', time.time() - t0) + + +class OffloadHook(accelerate.hooks.ModelHook): + def init_hook(self, module): + return module + + def pre_forward(self, module, *args, **kwargs): + if devices.normalize_device(module.device) != devices.normalize_device(devices.device): + device_index = torch.device(devices.device).index + if device_index is None: + device_index = 0 + max_memory = { + device_index: f"{shared.opts.diffusers_offload_max_gpu_memory}GiB", + "cpu": f"{shared.opts.diffusers_offload_max_cpu_memory}GiB", + } + device_map = accelerate.infer_auto_device_map(module, max_memory=max_memory) + module = accelerate.hooks.remove_hook_from_module(module, recurse=True) + offload_dir = getattr(module, "offload_dir", os.path.join(shared.opts.accelerate_offload_path, module.__class__.__name__)) + module = accelerate.dispatch_model(module, device_map=device_map, offload_dir=offload_dir) + module = accelerate.hooks.add_hook_to_module(module, OffloadHook(), append=True) + module._hf_hook.execution_device = torch.device(devices.device) # pylint: disable=protected-access + return args, kwargs + + def post_forward(self, module, output): + return output + + def detach_hook(self, module): + return module + + +offload_hook_instance = OffloadHook() def apply_balanced_offload(sd_model): - from accelerate import infer_auto_device_map, dispatch_model - from accelerate.hooks import add_hook_to_module, remove_hook_from_module, ModelHook + t0 = time.time() excluded = ['OmniGenPipeline'] if sd_model.__class__.__name__ in excluded: return sd_model - - class dispatch_from_cpu_hook(ModelHook): - def init_hook(self, module): - return module - - def pre_forward(self, module, *args, **kwargs): - if devices.normalize_device(module.device) != devices.normalize_device(devices.device): - device_index = torch.device(devices.device).index - if device_index is None: - device_index = 0 - max_memory = { - device_index: f"{shared.opts.diffusers_offload_max_gpu_memory}GiB", - "cpu": f"{shared.opts.diffusers_offload_max_cpu_memory}GiB", - } - device_map = infer_auto_device_map(module, max_memory=max_memory) - module = remove_hook_from_module(module, recurse=True) - offload_dir = getattr(module, "offload_dir", os.path.join(shared.opts.accelerate_offload_path, module.__class__.__name__)) - module = dispatch_model(module, device_map=device_map, offload_dir=offload_dir) - module = add_hook_to_module(module, dispatch_from_cpu_hook(), append=True) - module._hf_hook.execution_device = torch.device(devices.device) # pylint: disable=protected-access - return args, kwargs - - def post_forward(self, module, output): - return output - - def detach_hook(self, module): - return module + fn = f'{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access + debug_move(f'Apply offload: type=balanced fn={fn}') + checkpoint_name = sd_model.sd_checkpoint_info.name if getattr(sd_model, "sd_checkpoint_info", None) is not None else None + if checkpoint_name is None: + checkpoint_name = sd_model.__class__.__name__ def apply_balanced_offload_to_module(pipe): if hasattr(pipe, "pipe"): @@ -409,23 +420,19 @@ def apply_balanced_offload_to_module(pipe): for module_name in keys: # pylint: disable=protected-access module = getattr(pipe, module_name, None) if isinstance(module, torch.nn.Module): - checkpoint_name = pipe.sd_checkpoint_info.name if getattr(pipe, "sd_checkpoint_info", None) is not None else None - if checkpoint_name is None: - checkpoint_name = pipe.__class__.__name__ - offload_dir = os.path.join(shared.opts.accelerate_offload_path, checkpoint_name, module_name) network_layer_name = getattr(module, "network_layer_name", None) - module = remove_hook_from_module(module, recurse=True) + module = accelerate.hooks.remove_hook_from_module(module, recurse=True) try: - module = module.to("cpu") - module.offload_dir = offload_dir - module = add_hook_to_module(module, dispatch_from_cpu_hook(), append=True) + module = module.to(devices.cpu, non_blocking=True) + module.offload_dir = os.path.join(shared.opts.accelerate_offload_path, checkpoint_name, module_name) + # module = accelerate.hooks.add_hook_to_module(module, OffloadHook(), append=True) + module = accelerate.hooks.add_hook_to_module(module, offload_hook_instance, append=True) module._hf_hook.execution_device = torch.device(devices.device) # pylint: disable=protected-access if network_layer_name: module.network_layer_name = network_layer_name except Exception as e: if 'bitsandbytes' not in str(e): shared.log.error(f'Balanced offload: module={module_name} {e}') - devices.torch_gc(fast=True) apply_balanced_offload_to_module(sd_model) if hasattr(sd_model, "pipe"): @@ -435,6 +442,8 @@ def apply_balanced_offload_to_module(pipe): if hasattr(sd_model, "decoder_pipe"): apply_balanced_offload_to_module(sd_model.decoder_pipe) set_accelerate(sd_model) + devices.torch_gc(fast=True) + process_timer.add('offload', time.time() - t0) return sd_model From 2e244637b91f04eb615d47fef4066933f9e10908 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sat, 30 Nov 2024 20:49:57 +0300 Subject: [PATCH 030/249] Improve balanced offload pre forward performance --- modules/model_stablecascade.py | 8 +++----- modules/sd_models.py | 18 ++++++++++++------ 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/modules/model_stablecascade.py b/modules/model_stablecascade.py index 6c23ea00a..d6f9e4266 100644 --- a/modules/model_stablecascade.py +++ b/modules/model_stablecascade.py @@ -330,14 +330,12 @@ def __call__( elif output_type == "pil": images = images.permute(0, 2, 3, 1).cpu().float().numpy() # float() as bfloat16-> numpy doesnt work images = self.numpy_to_pil(images) + if shared.opts.diffusers_offload_mode == "balanced": + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) else: images = latents - # Offload all models - if shared.opts.diffusers_offload_mode == "balanced": - shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) - else: - self.maybe_free_model_hooks() + self.maybe_free_model_hooks() if not return_dict: return images diff --git a/modules/sd_models.py b/modules/sd_models.py index 2cf7b3931..c2c789987 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -378,15 +378,17 @@ def pre_forward(self, module, *args, **kwargs): if device_index is None: device_index = 0 max_memory = { - device_index: f"{shared.opts.diffusers_offload_max_gpu_memory}GiB", - "cpu": f"{shared.opts.diffusers_offload_max_cpu_memory}GiB", + device_index: int(shared.opts.diffusers_offload_max_gpu_memory * 1024*1024*1024), + "cpu": int(shared.opts.diffusers_offload_max_cpu_memory * 1024*1024*1024), } - device_map = accelerate.infer_auto_device_map(module, max_memory=max_memory) - module = accelerate.hooks.remove_hook_from_module(module, recurse=True) + device_map = getattr(module, "balanced_offload_device_map", None) + if device_map is None or max_memory != getattr(module, "balanced_offload_max_memory", None): + device_map = accelerate.infer_auto_device_map(module, max_memory=max_memory) offload_dir = getattr(module, "offload_dir", os.path.join(shared.opts.accelerate_offload_path, module.__class__.__name__)) module = accelerate.dispatch_model(module, device_map=device_map, offload_dir=offload_dir) - module = accelerate.hooks.add_hook_to_module(module, OffloadHook(), append=True) module._hf_hook.execution_device = torch.device(devices.device) # pylint: disable=protected-access + module.balanced_offload_device_map = device_map + module.balanced_offload_max_memory = max_memory return args, kwargs def post_forward(self, module, output): @@ -421,15 +423,19 @@ def apply_balanced_offload_to_module(pipe): module = getattr(pipe, module_name, None) if isinstance(module, torch.nn.Module): network_layer_name = getattr(module, "network_layer_name", None) + device_map = getattr(module, "balanced_offload_device_map", None) + max_memory = getattr(module, "balanced_offload_max_memory", None) module = accelerate.hooks.remove_hook_from_module(module, recurse=True) try: module = module.to(devices.cpu, non_blocking=True) module.offload_dir = os.path.join(shared.opts.accelerate_offload_path, checkpoint_name, module_name) - # module = accelerate.hooks.add_hook_to_module(module, OffloadHook(), append=True) module = accelerate.hooks.add_hook_to_module(module, offload_hook_instance, append=True) module._hf_hook.execution_device = torch.device(devices.device) # pylint: disable=protected-access if network_layer_name: module.network_layer_name = network_layer_name + if device_map and max_memory: + module.balanced_offload_device_map = device_map + module.balanced_offload_max_memory = max_memory except Exception as e: if 'bitsandbytes' not in str(e): shared.log.error(f'Balanced offload: module={module_name} {e}') From 8347c8055a51040ca2c22e05cbb35261b6d9e27e Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sat, 30 Nov 2024 21:12:26 +0300 Subject: [PATCH 031/249] Skip apply_balanced_offload if not needed --- modules/sd_models.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/modules/sd_models.py b/modules/sd_models.py index c2c789987..6c3ddc6b5 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -389,6 +389,7 @@ def pre_forward(self, module, *args, **kwargs): module._hf_hook.execution_device = torch.device(devices.device) # pylint: disable=protected-access module.balanced_offload_device_map = device_map module.balanced_offload_max_memory = max_memory + module.balanced_offload_active = True return args, kwargs def post_forward(self, module, output): @@ -421,7 +422,8 @@ def apply_balanced_offload_to_module(pipe): keys = get_signature(pipe).keys() for module_name in keys: # pylint: disable=protected-access module = getattr(pipe, module_name, None) - if isinstance(module, torch.nn.Module): + balanced_offload_active = getattr(module, "balanced_offload_active", None) + if isinstance(module, torch.nn.Module) and (balanced_offload_active is None or balanced_offload_active): network_layer_name = getattr(module, "network_layer_name", None) device_map = getattr(module, "balanced_offload_device_map", None) max_memory = getattr(module, "balanced_offload_max_memory", None) @@ -431,6 +433,7 @@ def apply_balanced_offload_to_module(pipe): module.offload_dir = os.path.join(shared.opts.accelerate_offload_path, checkpoint_name, module_name) module = accelerate.hooks.add_hook_to_module(module, offload_hook_instance, append=True) module._hf_hook.execution_device = torch.device(devices.device) # pylint: disable=protected-access + module.balanced_offload_active = False if network_layer_name: module.network_layer_name = network_layer_name if device_map and max_memory: @@ -1471,6 +1474,8 @@ def disable_offload(sd_model): module = getattr(sd_model, module_name, None) if isinstance(module, torch.nn.Module): network_layer_name = getattr(module, "network_layer_name", None) + if getattr(module, "balanced_offload_active", None) is not None: + module.balanced_offload_active = None module = remove_hook_from_module(module, recurse=True) if network_layer_name: module.network_layer_name = network_layer_name From 902bc9129c0ab1956a0295e4684fdc7318cddd1f Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 30 Nov 2024 17:26:49 -0500 Subject: [PATCH 032/249] update lora apply weights and xyz Signed-off-by: Vladimir Mandic --- modules/lora/networks.py | 81 ++++++++++++++++++++++------------------ modules/processing.py | 3 +- scripts/xyz_grid_on.py | 1 + 3 files changed, 48 insertions(+), 37 deletions(-) diff --git a/modules/lora/networks.py b/modules/lora/networks.py index 86c6e5ed0..b06a0c81f 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -27,8 +27,9 @@ available_networks = {} available_network_aliases = {} loaded_networks: List[network.Network] = [] -timer = { 'list': 0, 'load': 0, 'backup': 0, 'calc': 0, 'apply': 0, 'restore': 0, 'deactivate': 0 } +timer = { 'list': 0, 'load': 0, 'backup': 0, 'calc': 0, 'apply': 0, 'move': 0, 'restore': 0, 'deactivate': 0 } backup_size = 0 +bnb = None lora_cache = {} diffuser_loaded = [] diffuser_scales = [] @@ -302,42 +303,41 @@ def set_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm bias_backup = getattr(self, "network_bias_backup", None) if weights_backup is None and bias_backup is None: return - device = self.weight.device - with devices.inference_context(): - if weights_backup is not None: - if updown is not None: - if len(weights_backup.shape) == 4 and weights_backup.shape[1] == 9: # inpainting model. zero pad updown to make channel[1] 4 to 9 - updown = torch.nn.functional.pad(updown, (0, 0, 0, 0, 0, 5)) # pylint: disable=not-callable - weights_backup = weights_backup.clone().to(self.weight.device) - weights_backup += updown.to(weights_backup) - if getattr(self, "quant_type", None) in ['nf4', 'fp4']: - bnb = model_quant.load_bnb('Load network: type=LoRA', silent=True) - if bnb is not None: - self.weight = bnb.nn.Params4bit(weights_backup, quant_state=self.quant_state, quant_type=self.quant_type, blocksize=self.blocksize) - else: - self.weight.copy_(weights_backup, non_blocking=True) + if weights_backup is not None: + if updown is not None and len(weights_backup.shape) == 4 and weights_backup.shape[1] == 9: # inpainting model. zero pad updown to make channel[1] 4 to 9 + updown = torch.nn.functional.pad(updown, (0, 0, 0, 0, 0, 5)) # pylint: disable=not-callable + if updown is not None: + new_weight = updown.to(devices.device) + weights_backup.to(devices.device) + if getattr(self, "quant_type", None) in ['nf4', 'fp4'] and bnb is not None: + self.weight = bnb.nn.Params4bit(new_weight, quant_state=self.quant_state, quant_type=self.quant_type, blocksize=self.blocksize) else: - self.weight.copy_(weights_backup, non_blocking=True) - if hasattr(self, "qweight") and hasattr(self, "freeze"): - self.freeze() - if bias_backup is not None: - if ex_bias is not None: - bias_backup = bias_backup.clone() + ex_bias.to(weights_backup) - self.bias.copy_(bias_backup) + self.weight.copy_(new_weight, non_blocking=True) + del new_weight else: - self.bias = None - self.to(device) + self.weight.copy_(weights_backup, non_blocking=True) + if hasattr(self, "qweight") and hasattr(self, "freeze"): + self.freeze() + if bias_backup is not None: + if ex_bias is not None: + new_weight = ex_bias.to(self.bias.device) + bias_backup.to(self.device) + self.bias.copy_(new_weight, non_blocking=True) + del new_weight + else: + self.bias.copy_(bias_backup, non_blocking=True) + else: + self.bias = None t1 = time.time() timer['apply'] += t1 - t0 def maybe_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv], wanted_names): # pylint: disable=W0613 - global backup_size # pylint: disable=W0603 + global bnb, backup_size # pylint: disable=W0603 t0 = time.time() weights_backup = getattr(self, "network_weights_backup", None) if weights_backup is None and wanted_names != (): # pylint: disable=C1803 if getattr(self.weight, "quant_type", None) in ['nf4', 'fp4']: - bnb = model_quant.load_bnb('Load network: type=LoRA', silent=True) + if bnb is None: + bnb = model_quant.load_bnb('Load network: type=LoRA', silent=True) if bnb is not None: with devices.inference_context(): weights_backup = bnb.functional.dequantize_4bit(self.weight, quant_state=self.weight.quant_state, quant_type=self.weight.quant_type, blocksize=self.weight.blocksize,) @@ -375,21 +375,27 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn """ network_layer_name = getattr(self, 'network_layer_name', None) current_names = getattr(self, "network_current_names", ()) - wanted_names = tuple((x.name, x.te_multiplier, x.unet_multiplier, x.dyn_dim) for x in loaded_networks) + wanted_names = tuple((x.name, x.te_multiplier, x.unet_multiplier, x.dyn_dim) for x in loaded_networks) if len(loaded_networks) > 0 else () with devices.inference_context(): - if network_layer_name is not None and any([net.modules.get(network_layer_name, None) for net in loaded_networks]): # noqa: C419 + if len(loaded_networks) > 0 and network_layer_name is not None and any([net.modules.get(network_layer_name, None) for net in loaded_networks]): # noqa: C419 maybe_backup_weights(self, wanted_names) if current_names != wanted_names: + if shared.opts.diffusers_offload_mode == "none": + self.to(devices.device, non_blocking=True) batch_updown = None batch_ex_bias = None - t0 = time.time() for net in loaded_networks: - # default workflow where module is known and has weights module = net.modules.get(network_layer_name, None) if module is not None and hasattr(self, 'weight'): try: - weight = self.weight.to(devices.device) # calculate quant weights once + t0 = time.time() + weight = self.weight.to(devices.device, non_blocking=True) # calculate quant weights once + t1 = time.time() updown, ex_bias = module.calc_updown(weight) + del weight + t2 = time.time() + timer['move'] += t1 - t0 + timer['calc'] += t2 - t1 if batch_updown is not None and updown is not None: batch_updown += updown else: @@ -399,10 +405,13 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn else: batch_ex_bias = ex_bias if shared.opts.diffusers_offload_mode != "none": + t0 = time.time() if batch_updown is not None: - batch_updown = batch_updown.to(devices.cpu) + batch_updown = batch_updown.to(devices.cpu, non_blocking=True) if batch_ex_bias is not None: - batch_ex_bias = batch_ex_bias.to(devices.cpu) + batch_ex_bias = batch_ex_bias.to(devices.cpu, non_blocking=True) + t1 = time.time() + timer['move'] += t1 - t0 except RuntimeError as e: extra_network_lora.errors[net.name] = extra_network_lora.errors.get(net.name, 0) + 1 if debug: @@ -415,16 +424,16 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn continue shared.log.warning(f'LoRA network="{net.name}" layer="{network_layer_name}" unsupported operation') extra_network_lora.errors[net.name] = extra_network_lora.errors.get(net.name, 0) + 1 - t1 = time.time() - timer['calc'] += t1 - t0 set_weights(self, batch_updown, batch_ex_bias) # Set or restore weights from backup self.network_current_names = wanted_names + # self.to(devices.cpu) -def network_load(): # called from processing +def network_load(): timer['backup'] = 0 timer['calc'] = 0 timer['apply'] = 0 + timer['move'] = 0 sd_model = getattr(shared.sd_model, "pipe", shared.sd_model) # wrapped model compatiblility if shared.opts.diffusers_offload_mode == "sequential": sd_models.disable_offload(sd_model) diff --git a/modules/processing.py b/modules/processing.py index 92faaee8d..ebbaf7272 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -472,7 +472,8 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: if p.scripts is not None and isinstance(p.scripts, scripts.ScriptRunner) and not (shared.state.interrupted or shared.state.skipped): p.scripts.postprocess(p, processed) timer.process.record('post') - shared.log.info(f'Processed: images={len(output_images)} its={(p.steps * len(output_images)) / (t1 - t0):.2f} time={t1-t0:.2f} timers={timer.process.dct(min_time=0.02)} memory={memstats.memory_stats()}') + if not p.disable_extra_networks: + shared.log.info(f'Processed: images={len(output_images)} its={(p.steps * len(output_images)) / (t1 - t0):.2f} time={t1-t0:.2f} timers={timer.process.dct(min_time=0.02)} memory={memstats.memory_stats()}') if shared.cmd_opts.malloc: import tracemalloc diff --git a/scripts/xyz_grid_on.py b/scripts/xyz_grid_on.py index 202a2cfc4..aa0897442 100644 --- a/scripts/xyz_grid_on.py +++ b/scripts/xyz_grid_on.py @@ -413,6 +413,7 @@ def cell(x, y, z, ix, iy, iz): p.do_not_save_grid = True p.do_not_save_samples = True + p.disable_extra_networks = True active = False cache = processed return processed From 7003024d1467b7e57f5a6d226c4f8d0404523155 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 30 Nov 2024 18:10:10 -0500 Subject: [PATCH 033/249] update stats Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 3 +++ modules/lora/networks.py | 42 +++++++++++++++++++++++++-------------- modules/processing_vae.py | 2 +- 3 files changed, 31 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dcb88bcf3..9ab3bcbd1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,8 @@ - Flux: all-in-one safetensors example: - Flux: do not recast quants +- **Offload** improvements: + - faster and more compatible *balanced* mode - **UI**: - improved stats on generate completion - improved live preview display and performance @@ -69,6 +71,7 @@ - fix gpu memory monitoring - simplify img2img/inpaint/sketch canvas handling - fix prompt caching +- fix xyz grid skip final pass ## Update for 2024-11-21 diff --git a/modules/lora/networks.py b/modules/lora/networks.py index b06a0c81f..604e591a9 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -3,9 +3,10 @@ import re import time import concurrent +from contextlib import nullcontext import torch import diffusers.models.lora -import rich.progress as p +import rich.progress as rp import modules.lora.network as network import modules.lora.network_lora as network_lora @@ -22,7 +23,6 @@ debug = os.environ.get('SD_LORA_DEBUG', None) is not None -pbar = p.Progress(p.TextColumn('[cyan]{task.description}'), p.BarColumn(), p.TaskProgressColumn(), p.TimeRemainingColumn(), p.TimeElapsedColumn(), console=shared.console) extra_network_lora = None available_networks = {} available_network_aliases = {} @@ -307,7 +307,7 @@ def set_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm if updown is not None and len(weights_backup.shape) == 4 and weights_backup.shape[1] == 9: # inpainting model. zero pad updown to make channel[1] 4 to 9 updown = torch.nn.functional.pad(updown, (0, 0, 0, 0, 0, 5)) # pylint: disable=not-callable if updown is not None: - new_weight = updown.to(devices.device) + weights_backup.to(devices.device) + new_weight = updown.to(devices.device, non_blocking=True) + weights_backup.to(devices.device, non_blocking=True) if getattr(self, "quant_type", None) in ['nf4', 'fp4'] and bnb is not None: self.weight = bnb.nn.Params4bit(new_weight, quant_state=self.quant_state, quant_type=self.quant_type, blocksize=self.blocksize) else: @@ -319,7 +319,7 @@ def set_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm self.freeze() if bias_backup is not None: if ex_bias is not None: - new_weight = ex_bias.to(self.bias.device) + bias_backup.to(self.device) + new_weight = ex_bias.to(devices.device, non_blocking=True) + bias_backup.to(devices.device, non_blocking=True) self.bias.copy_(new_weight, non_blocking=True) del new_weight else: @@ -351,7 +351,6 @@ def maybe_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn. if shared.opts.lora_offload_backup and weights_backup is not None: weights_backup = weights_backup.to(devices.cpu) self.network_weights_backup = weights_backup - backup_size += weights_backup.numel() * weights_backup.element_size() bias_backup = getattr(self, "network_bias_backup", None) if bias_backup is None: if getattr(self, 'bias', None) is not None: @@ -361,8 +360,10 @@ def maybe_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn. if shared.opts.lora_offload_backup and bias_backup is not None: bias_backup = bias_backup.to(devices.cpu) self.network_bias_backup = bias_backup - if bias_backup is not None: - backup_size += bias_backup.numel() * bias_backup.element_size() + if getattr(self, 'network_weights_backup', None) is not None: + backup_size += self.network_weights_backup.numel() * self.network_weights_backup.element_size() + if getattr(self, 'network_bias_backup', None) is not None: + backup_size += self.network_bias_backup.numel() * self.network_bias_backup.element_size() t1 = time.time() timer['backup'] += t1 - t0 @@ -424,9 +425,11 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn continue shared.log.warning(f'LoRA network="{net.name}" layer="{network_layer_name}" unsupported operation') extra_network_lora.errors[net.name] = extra_network_lora.errors.get(net.name, 0) + 1 - set_weights(self, batch_updown, batch_ex_bias) # Set or restore weights from backup self.network_current_names = wanted_names - # self.to(devices.cpu) + set_weights(self, batch_updown, batch_ex_bias) # Set or restore weights from backup + if batch_updown is not None or batch_ex_bias is not None: + return self.weight.device + return None def network_load(): @@ -443,15 +446,24 @@ def network_load(): component = getattr(sd_model, component_name, None) if component is not None and hasattr(component, 'named_modules'): modules += list(component.named_modules()) + devices_used = [] + if len(loaded_networks) > 0: + pbar = rp.Progress(rp.TextColumn('[cyan]{task.description}'), rp.BarColumn(), rp.TaskProgressColumn(), rp.TimeRemainingColumn(), rp.TimeElapsedColumn(), console=shared.console) + task = pbar.add_task(description='Apply network: type=LoRA' , total=len(modules)) + else: + task = None + pbar = nullcontext() with pbar: - task = pbar.add_task(description='Apply network: type=LoRA' , total=len(modules), visible=len(loaded_networks) > 0) for _, module in modules: - network_apply_weights(module) - pbar.update(task, advance=1) # progress bar becomes visible if operation takes more than 1sec - pbar.remove_task(task) - modules.clear() + devices_used.append(network_apply_weights(module)) + if task is not None: + pbar.update(task, advance=1) # progress bar becomes visible if operation takes more than 1sec + # pbar.remove_task(task) if debug: - shared.log.debug(f'Load network: type=LoRA modules={len(modules)} backup={backup_size} time={get_timers()}') + devices_used = [d for d in devices_used if d is not None] + devices_set = list(set(devices_used)) + shared.log.debug(f'Load network: type=LoRA modules={len(modules)} apply={len(devices_used)} device={devices_set} backup={backup_size} time={get_timers()}') + modules.clear() if shared.opts.diffusers_offload_mode == "sequential": sd_models.set_diffuser_offload(sd_model, op="model") diff --git a/modules/processing_vae.py b/modules/processing_vae.py index 1c4a45f07..b114e01d3 100644 --- a/modules/processing_vae.py +++ b/modules/processing_vae.py @@ -117,7 +117,7 @@ def full_vae_decode(latents, model): model.vae.orig_dtype = model.vae.dtype model.vae = model.vae.to(dtype=torch.float32) latents = latents.to(torch.float32) - latents = latents.to(devices.device) + latents = latents.to(devices.device, non_blocking=True) if getattr(model.vae, "post_quant_conv", None) is not None: latents = latents.to(next(iter(model.vae.post_quant_conv.parameters())).dtype) From 6413c1fd0675d0e34958e264e9b6ffddc4d038fa Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 30 Nov 2024 19:03:51 -0500 Subject: [PATCH 034/249] add low/high threshold to balanced offload Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 2 ++ modules/devices.py | 3 ++- modules/lora/networks.py | 1 + modules/processing_helpers.py | 5 ++--- modules/sd_models.py | 27 ++++++++++++++++++++------- modules/shared.py | 5 +++-- 6 files changed, 30 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ab3bcbd1..c62b6b917 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,8 @@ - Flux: do not recast quants - **Offload** improvements: - faster and more compatible *balanced* mode + - balanced offload: units are now in percentage instead of bytes + - balanced offload: add both high and low watermark - **UI**: - improved stats on generate completion - improved live preview display and performance diff --git a/modules/devices.py b/modules/devices.py index 9ca1863a5..64968a30c 100644 --- a/modules/devices.py +++ b/modules/devices.py @@ -224,7 +224,7 @@ def torch_gc(force=False, fast=False): timer.process.records['gc'] = 0 timer.process.records['gc'] += t1 - t0 if not force or collected == 0: - return + return used_gpu mem = memstats.memory_stats() saved = round(gpu.get('used', 0) - mem.get('gpu', {}).get('used', 0), 2) before = { 'gpu': gpu.get('used', 0), 'ram': ram.get('used', 0) } @@ -233,6 +233,7 @@ def torch_gc(force=False, fast=False): results = { 'collected': collected, 'saved': saved } fn = f'{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access log.debug(f'GC: utilization={utilization} gc={results} before={before} after={after} device={torch.device(get_optimal_device_name())} fn={fn} time={round(t1 - t0, 2)}') # pylint: disable=protected-access + return used_gpu def set_cuda_sync_mode(mode): diff --git a/modules/lora/networks.py b/modules/lora/networks.py index 604e591a9..69db5fce3 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -218,6 +218,7 @@ def maybe_recompile_model(names, te_multipliers): def load_networks(names, te_multipliers=None, unet_multipliers=None, dyn_dims=None): + timer['list'] = 0 global backup_size # pylint: disable=global-statement networks_on_disk: list[network.NetworkOnDisk] = [available_network_aliases.get(name, None) for name in names] if any(x is None for x in networks_on_disk): diff --git a/modules/processing_helpers.py b/modules/processing_helpers.py index ab08d4cc8..5d2661cc2 100644 --- a/modules/processing_helpers.py +++ b/modules/processing_helpers.py @@ -368,14 +368,13 @@ def validate_sample(tensor): sample = 255.0 * np.moveaxis(sample, 0, 2) if not shared.native else 255.0 * sample with warnings.catch_warnings(record=True) as w: cast = sample.astype(np.uint8) - minimum, maximum, mean = np.min(cast), np.max(cast), np.mean(cast) - if len(w) > 0 or minimum == maximum: + if len(w) > 0: nans = np.isnan(sample).sum() cast = np.nan_to_num(sample) cast = cast.astype(np.uint8) vae = shared.sd_model.vae.dtype if hasattr(shared.sd_model, 'vae') else None upcast = getattr(shared.sd_model.vae.config, 'force_upcast', None) if hasattr(shared.sd_model, 'vae') and hasattr(shared.sd_model.vae, 'config') else None - shared.log.error(f'Decode: sample={sample.shape} invalid={nans} mean={mean} dtype={dtype} vae={vae} upcast={upcast} failed to validate') + shared.log.error(f'Decode: sample={sample.shape} invalid={nans} dtype={dtype} vae={vae} upcast={upcast} failed to validate') if upcast is not None and not upcast: setattr(shared.sd_model.vae.config, 'force_upcast', True) # noqa: B010 shared.log.warning('Decode: upcast=True set, retry operation') diff --git a/modules/sd_models.py b/modules/sd_models.py index 6c3ddc6b5..42bd33d82 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -361,7 +361,7 @@ def set_diffuser_offload(sd_model, op: str = 'model'): shared.log.error(f'Setting {op}: offload={shared.opts.diffusers_offload_mode} {e}') if shared.opts.diffusers_offload_mode == "balanced": try: - shared.log.debug(f'Setting {op}: offload={shared.opts.diffusers_offload_mode} threshold={shared.opts.diffusers_offload_max_gpu_memory} limit={shared.opts.cuda_mem_fraction}') + shared.log.debug(f'Setting {op}: offload={shared.opts.diffusers_offload_mode} watermarks low={shared.opts.diffusers_offload_min_gpu_memory} high={shared.opts.diffusers_offload_max_gpu_memory} limit={shared.opts.cuda_mem_fraction:.2f}') sd_model = apply_balanced_offload(sd_model) except Exception as e: shared.log.error(f'Setting {op}: offload={shared.opts.diffusers_offload_mode} {e}') @@ -369,6 +369,16 @@ def set_diffuser_offload(sd_model, op: str = 'model'): class OffloadHook(accelerate.hooks.ModelHook): + def __init__(self): + if shared.opts.diffusers_offload_max_gpu_memory > 1: + shared.opts.diffusers_offload_max_gpu_memory = 0.75 + if shared.opts.diffusers_offload_max_cpu_memory > 1: + shared.opts.diffusers_offload_max_cpu_memory = 0.75 + self.gpu = int(shared.gpu_memory * shared.opts.diffusers_offload_max_gpu_memory * 1024*1024*1024) + self.cpu = int(shared.cpu_memory * shared.opts.diffusers_offload_max_cpu_memory * 1024*1024*1024) + shared.log.info(f'Init offload: type=balanced gpu={self.gpu} cpu={self.cpu}') + super().__init__() + def init_hook(self, module): return module @@ -377,10 +387,7 @@ def pre_forward(self, module, *args, **kwargs): device_index = torch.device(devices.device).index if device_index is None: device_index = 0 - max_memory = { - device_index: int(shared.opts.diffusers_offload_max_gpu_memory * 1024*1024*1024), - "cpu": int(shared.opts.diffusers_offload_max_cpu_memory * 1024*1024*1024), - } + max_memory = { device_index: self.gpu, "cpu": self.cpu } device_map = getattr(module, "balanced_offload_device_map", None) if device_map is None or max_memory != getattr(module, "balanced_offload_max_memory", None): device_map = accelerate.infer_auto_device_map(module, max_memory=max_memory) @@ -399,10 +406,13 @@ def detach_hook(self, module): return module -offload_hook_instance = OffloadHook() +offload_hook_instance = None def apply_balanced_offload(sd_model): + global offload_hook_instance # pylint: disable=global-statement + if offload_hook_instance is None: + offload_hook_instance = OffloadHook() t0 = time.time() excluded = ['OmniGenPipeline'] if sd_model.__class__.__name__ in excluded: @@ -414,6 +424,7 @@ def apply_balanced_offload(sd_model): checkpoint_name = sd_model.__class__.__name__ def apply_balanced_offload_to_module(pipe): + used_gpu = devices.torch_gc(fast=True) if hasattr(pipe, "pipe"): apply_balanced_offload_to_module(pipe.pipe) if hasattr(pipe, "_internal_dict"): @@ -429,7 +440,9 @@ def apply_balanced_offload_to_module(pipe): max_memory = getattr(module, "balanced_offload_max_memory", None) module = accelerate.hooks.remove_hook_from_module(module, recurse=True) try: - module = module.to(devices.cpu, non_blocking=True) + if used_gpu > 100 * shared.opts.diffusers_offload_min_gpu_memory: + module = module.to(devices.cpu, non_blocking=True) + used_gpu = devices.torch_gc(fast=True) module.offload_dir = os.path.join(shared.opts.accelerate_offload_path, checkpoint_name, module_name) module = accelerate.hooks.add_hook_to_module(module, offload_hook_instance, append=True) module._hf_hook.execution_device = torch.device(devices.device) # pylint: disable=protected-access diff --git a/modules/shared.py b/modules/shared.py index ea6a44a3b..b8fac60ab 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -560,8 +560,9 @@ def get_default_modes(): "diffusers_extract_ema": OptionInfo(False, "Use model EMA weights when possible"), "diffusers_generator_device": OptionInfo("GPU", "Generator device", gr.Radio, {"choices": ["GPU", "CPU", "Unset"]}), "diffusers_offload_mode": OptionInfo(startup_offload_mode, "Model offload mode", gr.Radio, {"choices": ['none', 'balanced', 'model', 'sequential']}), - "diffusers_offload_max_gpu_memory": OptionInfo(round(gpu_memory * 0.75, 1), "Max GPU memory before balanced offload", gr.Slider, {"minimum": 0, "maximum": gpu_memory, "step": 0.01, "visible": True }), - "diffusers_offload_max_cpu_memory": OptionInfo(round(cpu_memory * 0.75, 1), "Max CPU memory before balanced offload", gr.Slider, {"minimum": 0, "maximum": cpu_memory, "step": 0.01, "visible": False }), + "diffusers_offload_min_gpu_memory": OptionInfo(0.25, "Balanced offload GPU low watermark", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01 }), + "diffusers_offload_max_gpu_memory": OptionInfo(0.75, "Balanced offload GPU high watermark", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01 }), + "diffusers_offload_max_cpu_memory": OptionInfo(0.75, "Balanced offload CPU high watermark", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01 }), "diffusers_vae_upcast": OptionInfo("default", "VAE upcasting", gr.Radio, {"choices": ['default', 'true', 'false']}), "diffusers_vae_slicing": OptionInfo(True, "VAE slicing"), "diffusers_vae_tiling": OptionInfo(cmd_opts.lowvram or cmd_opts.medvram, "VAE tiling"), From 4d8b39aa02b1d0151481b93ee278c59edc9c4b49 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 30 Nov 2024 19:15:50 -0500 Subject: [PATCH 035/249] reinit offoad instance on change Signed-off-by: Vladimir Mandic --- modules/sd_models.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/modules/sd_models.py b/modules/sd_models.py index 42bd33d82..24ceff5ee 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -374,9 +374,14 @@ def __init__(self): shared.opts.diffusers_offload_max_gpu_memory = 0.75 if shared.opts.diffusers_offload_max_cpu_memory > 1: shared.opts.diffusers_offload_max_cpu_memory = 0.75 + self.min_watermark = shared.opts.diffusers_offload_min_gpu_memory + self.max_watermark = shared.opts.diffusers_offload_max_gpu_memory + self.cpu_watermark = shared.opts.diffusers_offload_max_cpu_memory self.gpu = int(shared.gpu_memory * shared.opts.diffusers_offload_max_gpu_memory * 1024*1024*1024) self.cpu = int(shared.cpu_memory * shared.opts.diffusers_offload_max_cpu_memory * 1024*1024*1024) - shared.log.info(f'Init offload: type=balanced gpu={self.gpu} cpu={self.cpu}') + gpu_dict = { "min": self.min_watermark, "max": self.max_watermark, "bytes": self.gpu } + cpu_dict = { "max": self.cpu_watermark, "bytes": self.cpu } + shared.log.info(f'Init offload: type=balanced gpu={gpu_dict} cpu={cpu_dict}') super().__init__() def init_hook(self, module): @@ -411,7 +416,7 @@ def detach_hook(self, module): def apply_balanced_offload(sd_model): global offload_hook_instance # pylint: disable=global-statement - if offload_hook_instance is None: + if offload_hook_instance is None or offload_hook_instance.min_watermark != shared.opts.diffusers_offload_min_gpu_memory or offload_hook_instance.max_watermark != shared.opts.diffusers_offload_max_gpu_memory: offload_hook_instance = OffloadHook() t0 = time.time() excluded = ['OmniGenPipeline'] From 5cc849721fb8e3de2e587abb917ecb31d2145b74 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sun, 1 Dec 2024 03:52:57 +0300 Subject: [PATCH 036/249] Fix NaNs on Intel with Lora + Offloading --- modules/lora/networks.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/lora/networks.py b/modules/lora/networks.py index 69db5fce3..4312f3405 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -412,6 +412,9 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn batch_updown = batch_updown.to(devices.cpu, non_blocking=True) if batch_ex_bias is not None: batch_ex_bias = batch_ex_bias.to(devices.cpu, non_blocking=True) + if devices.backend == "ipex": + # using non_blocking=True here causes NaNs on Intel + torch.xpu.synchronize(devices.device) t1 = time.time() timer['move'] += t1 - t0 except RuntimeError as e: From d703359219d44fe9c6a32343fda67c1c97f41bc8 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 30 Nov 2024 19:53:08 -0500 Subject: [PATCH 037/249] interruptible lora apply Signed-off-by: Vladimir Mandic --- modules/lora/networks.py | 2 ++ scripts/xyz_grid_shared.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/lora/networks.py b/modules/lora/networks.py index 4312f3405..83a62eb5a 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -459,6 +459,8 @@ def network_load(): pbar = nullcontext() with pbar: for _, module in modules: + if shared.state.interrupted: + continue devices_used.append(network_apply_weights(module)) if task is not None: pbar.update(task, advance=1) # progress bar becomes visible if operation takes more than 1sec diff --git a/scripts/xyz_grid_shared.py b/scripts/xyz_grid_shared.py index 82387fab8..3fc8d32c8 100644 --- a/scripts/xyz_grid_shared.py +++ b/scripts/xyz_grid_shared.py @@ -192,7 +192,7 @@ def apply_vae(p, x, xs): def list_lora(): import sys - lora = [v for k, v in sys.modules.items() if k == 'networks'][0] + lora = [v for k, v in sys.modules.items() if k == 'networks' or k == 'modules.lora.networks'][0] loras = [v.fullname for v in lora.available_networks.values()] return ['None'] + loras From 5af1f6c5895ca588fca9f77c1849cd1ed7e25afa Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 1 Dec 2024 10:54:51 -0500 Subject: [PATCH 038/249] lora-refactor Signed-off-by: Vladimir Mandic --- modules/extra_networks.py | 5 +- modules/lora/extra_networks_lora.py | 5 +- modules/lora/lora_convert.py | 31 +++ modules/lora/lyco_helpers.py | 4 +- modules/lora/networks.py | 364 +++++++++++++--------------- scripts/xyz_grid.py | 1 + 6 files changed, 208 insertions(+), 202 deletions(-) diff --git a/modules/extra_networks.py b/modules/extra_networks.py index 010157af9..fca48e21c 100644 --- a/modules/extra_networks.py +++ b/modules/extra_networks.py @@ -17,8 +17,9 @@ def register_extra_network(extra_network): def register_default_extra_networks(): from modules.ui_extra_networks_styles import ExtraNetworkStyles register_extra_network(ExtraNetworkStyles()) - from modules.lora.extra_networks_lora import ExtraNetworkLora - register_extra_network(ExtraNetworkLora()) + if shared.native: + from modules.lora.networks import extra_network_lora + register_extra_network(extra_network_lora) if shared.opts.hypernetwork_enabled: from modules.ui_extra_networks_hypernet import ExtraNetworkHypernet register_extra_network(ExtraNetworkHypernet()) diff --git a/modules/lora/extra_networks_lora.py b/modules/lora/extra_networks_lora.py index c875ba0d5..d58cebd8f 100644 --- a/modules/lora/extra_networks_lora.py +++ b/modules/lora/extra_networks_lora.py @@ -4,6 +4,7 @@ import modules.lora.networks as networks from modules import extra_networks, shared + # from https://github.com/cheald/sd-webui-loractl/blob/master/loractl/lib/utils.py def get_stepwise(param, step, steps): def sorted_positions(raw_steps): @@ -122,8 +123,8 @@ def activate(self, p, params_list, step=0): self.active = True self.model = shared.opts.sd_model_checkpoint names, te_multipliers, unet_multipliers, dyn_dims = parse(p, params_list, step) - networks.load_networks(names, te_multipliers, unet_multipliers, dyn_dims) # load - networks.network_load() # backup/apply + networks.network_load(names, te_multipliers, unet_multipliers, dyn_dims) # load + networks.network_process() if len(networks.loaded_networks) > 0 and step == 0: infotext(p) prompt(p) diff --git a/modules/lora/lora_convert.py b/modules/lora/lora_convert.py index dc86a24cf..032ffa5a3 100644 --- a/modules/lora/lora_convert.py +++ b/modules/lora/lora_convert.py @@ -476,3 +476,34 @@ def _convert_sd_scripts_to_ai_toolkit(sds_sd): return new_state_dict return _convert_sd_scripts_to_ai_toolkit(state_dict) + + +def assign_network_names_to_compvis_modules(sd_model): + if sd_model is None: + return + sd_model = getattr(shared.sd_model, "pipe", shared.sd_model) # wrapped model compatiblility + network_layer_mapping = {} + if hasattr(sd_model, 'text_encoder') and sd_model.text_encoder is not None: + for name, module in sd_model.text_encoder.named_modules(): + prefix = "lora_te1_" if hasattr(sd_model, 'text_encoder_2') else "lora_te_" + network_name = prefix + name.replace(".", "_") + network_layer_mapping[network_name] = module + module.network_layer_name = network_name + if hasattr(sd_model, 'text_encoder_2'): + for name, module in sd_model.text_encoder_2.named_modules(): + network_name = "lora_te2_" + name.replace(".", "_") + network_layer_mapping[network_name] = module + module.network_layer_name = network_name + if hasattr(sd_model, 'unet'): + for name, module in sd_model.unet.named_modules(): + network_name = "lora_unet_" + name.replace(".", "_") + network_layer_mapping[network_name] = module + module.network_layer_name = network_name + if hasattr(sd_model, 'transformer'): + for name, module in sd_model.transformer.named_modules(): + network_name = "lora_transformer_" + name.replace(".", "_") + network_layer_mapping[network_name] = module + if "norm" in network_name and "linear" not in network_name and shared.sd_model_type != "sd3": + continue + module.network_layer_name = network_name + shared.sd_model.network_layer_mapping = network_layer_mapping diff --git a/modules/lora/lyco_helpers.py b/modules/lora/lyco_helpers.py index 9a16d25ab..ac4f2419f 100644 --- a/modules/lora/lyco_helpers.py +++ b/modules/lora/lyco_helpers.py @@ -12,13 +12,13 @@ def rebuild_conventional(up, down, shape, dyn_dim=None): if dyn_dim is not None: up = up[:, :dyn_dim] down = down[:dyn_dim, :] - return (up @ down).reshape(shape) + return (up @ down).reshape(shape).to(up.dtype) def rebuild_cp_decomposition(up, down, mid): up = up.reshape(up.size(0), -1) down = down.reshape(down.size(0), -1) - return torch.einsum('n m k l, i n, m j -> i j k l', mid, up, down) + return torch.einsum('n m k l, i n, m j -> i j k l', mid, up, down).to(up.dtype) # copied from https://github.com/KohakuBlueleaf/LyCORIS/blob/dev/lycoris/modules/lokr.py diff --git a/modules/lora/networks.py b/modules/lora/networks.py index 83a62eb5a..e0f2134c9 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -19,16 +19,16 @@ import modules.lora.network_glora as network_glora import modules.lora.network_overrides as network_overrides import modules.lora.lora_convert as lora_convert +from modules.lora.extra_networks_lora import ExtraNetworkLora from modules import shared, devices, sd_models, sd_models_compile, errors, files_cache, model_quant debug = os.environ.get('SD_LORA_DEBUG', None) is not None -extra_network_lora = None +extra_network_lora = ExtraNetworkLora() available_networks = {} available_network_aliases = {} loaded_networks: List[network.Network] = [] timer = { 'list': 0, 'load': 0, 'backup': 0, 'calc': 0, 'apply': 0, 'move': 0, 'restore': 0, 'deactivate': 0 } -backup_size = 0 bnb = None lora_cache = {} diffuser_loaded = [] @@ -60,36 +60,7 @@ def get_timers(): return t -def assign_network_names_to_compvis_modules(sd_model): - if sd_model is None: - return - sd_model = getattr(shared.sd_model, "pipe", shared.sd_model) # wrapped model compatiblility - network_layer_mapping = {} - if hasattr(sd_model, 'text_encoder') and sd_model.text_encoder is not None: - for name, module in sd_model.text_encoder.named_modules(): - prefix = "lora_te1_" if hasattr(sd_model, 'text_encoder_2') else "lora_te_" - network_name = prefix + name.replace(".", "_") - network_layer_mapping[network_name] = module - module.network_layer_name = network_name - if hasattr(sd_model, 'text_encoder_2'): - for name, module in sd_model.text_encoder_2.named_modules(): - network_name = "lora_te2_" + name.replace(".", "_") - network_layer_mapping[network_name] = module - module.network_layer_name = network_name - if hasattr(sd_model, 'unet'): - for name, module in sd_model.unet.named_modules(): - network_name = "lora_unet_" + name.replace(".", "_") - network_layer_mapping[network_name] = module - module.network_layer_name = network_name - if hasattr(sd_model, 'transformer'): - for name, module in sd_model.transformer.named_modules(): - network_name = "lora_transformer_" + name.replace(".", "_") - network_layer_mapping[network_name] = module - if "norm" in network_name and "linear" not in network_name and shared.sd_model_type != "sd3": - continue - module.network_layer_name = network_name - shared.sd_model.network_layer_mapping = network_layer_mapping - +# section: load networks from disk def load_diffusers(name, network_on_disk, lora_scale=shared.opts.extra_networks_default_multiplier) -> Union[network.Network, None]: name = name.replace(".", "_") @@ -120,7 +91,7 @@ def load_diffusers(name, network_on_disk, lora_scale=shared.opts.extra_networks_ return net -def load_network(name, network_on_disk) -> Union[network.Network, None]: +def load_safetensors(name, network_on_disk) -> Union[network.Network, None]: if not shared.sd_loaded: return None @@ -139,7 +110,7 @@ def load_network(name, network_on_disk) -> Union[network.Network, None]: sd = lora_convert._convert_kohya_sd3_lora_to_diffusers(sd) or sd # pylint: disable=protected-access except ValueError: # EAFP for diffusers PEFT keys pass - assign_network_names_to_compvis_modules(shared.sd_model) + lora_convert.assign_network_names_to_compvis_modules(shared.sd_model) keys_failed_to_match = {} matched_networks = {} bundle_embeddings = {} @@ -217,9 +188,46 @@ def maybe_recompile_model(names, te_multipliers): return recompile_model -def load_networks(names, te_multipliers=None, unet_multipliers=None, dyn_dims=None): +def list_available_networks(): + t0 = time.time() + available_networks.clear() + available_network_aliases.clear() + forbidden_network_aliases.clear() + available_network_hash_lookup.clear() + forbidden_network_aliases.update({"none": 1, "Addams": 1}) + if not os.path.exists(shared.cmd_opts.lora_dir): + shared.log.warning(f'LoRA directory not found: path="{shared.cmd_opts.lora_dir}"') + + def add_network(filename): + if not os.path.isfile(filename): + return + name = os.path.splitext(os.path.basename(filename))[0] + name = name.replace('.', '_') + try: + entry = network.NetworkOnDisk(name, filename) + available_networks[entry.name] = entry + if entry.alias in available_network_aliases: + forbidden_network_aliases[entry.alias.lower()] = 1 + if shared.opts.lora_preferred_name == 'filename': + available_network_aliases[entry.name] = entry + else: + available_network_aliases[entry.alias] = entry + if entry.shorthash: + available_network_hash_lookup[entry.shorthash] = entry + except OSError as e: # should catch FileNotFoundError and PermissionError etc. + shared.log.error(f'LoRA: filename="{filename}" {e}') + + candidates = list(files_cache.list_files(shared.cmd_opts.lora_dir, ext_filter=[".pt", ".ckpt", ".safetensors"])) + with concurrent.futures.ThreadPoolExecutor(max_workers=shared.max_workers) as executor: + for fn in candidates: + executor.submit(add_network, fn) + t1 = time.time() + timer['list'] = t1 - t0 + shared.log.info(f'Available LoRAs: path="{shared.cmd_opts.lora_dir}" items={len(available_networks)} folders={len(forbidden_network_aliases)} time={t1 - t0:.2f}') + + +def network_load(names, te_multipliers=None, unet_multipliers=None, dyn_dims=None): timer['list'] = 0 - global backup_size # pylint: disable=global-statement networks_on_disk: list[network.NetworkOnDisk] = [available_network_aliases.get(name, None) for name in names] if any(x is None for x in networks_on_disk): list_available_networks() @@ -244,7 +252,7 @@ def load_networks(names, te_multipliers=None, unet_multipliers=None, dyn_dims=No if shared.opts.lora_force_diffusers or network_overrides.check_override(shorthash): # OpenVINO only works with Diffusers LoRa loading net = load_diffusers(name, network_on_disk, lora_scale=te_multipliers[i] if te_multipliers else shared.opts.extra_networks_default_multiplier) else: - net = load_network(name, network_on_disk) + net = load_safetensors(name, network_on_disk) if net is not None: net.mentioned_name = name network_on_disk.read_hash() @@ -294,17 +302,108 @@ def load_networks(names, te_multipliers=None, unet_multipliers=None, dyn_dims=No devices.torch_gc() t1 = time.time() - backup_size = 0 timer['load'] = t1 - t0 -def set_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv], updown, ex_bias): +# section: process loaded networks + +def network_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv], weight, network_layer_name, wanted_names): + global bnb # pylint: disable=W0603 + backup_size = 0 + if len(loaded_networks) > 0 and network_layer_name is not None and any([net.modules.get(network_layer_name, None) for net in loaded_networks]): # noqa: C419 + t0 = time.time() + weights_backup = getattr(self, "network_weights_backup", None) + if weights_backup is None and wanted_names != (): # pylint: disable=C1803 + self.network_weights_backup = None + if getattr(weight, "quant_type", None) in ['nf4', 'fp4']: + if bnb is None: + bnb = model_quant.load_bnb('Load network: type=LoRA', silent=True) + if bnb is not None: + with devices.inference_context(): + weights_backup = bnb.functional.dequantize_4bit(weight, quant_state=weight.quant_state, quant_type=weight.quant_type, blocksize=weight.blocksize,) + self.quant_state = weight.quant_state + self.quant_type = weight.quant_type + self.blocksize = weight.blocksize + else: + weights_backup = weight.clone() + else: + weights_backup = weight.clone() + if shared.opts.lora_offload_backup and weights_backup is not None: + weights_backup = weights_backup.to(devices.cpu) + self.network_weights_backup = weights_backup + bias_backup = getattr(self, "network_bias_backup", None) + if bias_backup is None: + if getattr(self, 'bias', None) is not None: + bias_backup = self.bias.clone() + else: + bias_backup = None + if shared.opts.lora_offload_backup and bias_backup is not None: + bias_backup = bias_backup.to(devices.cpu) + self.network_bias_backup = bias_backup + if getattr(self, 'network_weights_backup', None) is not None: + backup_size += self.network_weights_backup.numel() * self.network_weights_backup.element_size() + if getattr(self, 'network_bias_backup', None) is not None: + backup_size += self.network_bias_backup.numel() * self.network_bias_backup.element_size() + t1 = time.time() + timer['backup'] += t1 - t0 + return backup_size + + +def network_calc_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv], weight, network_layer_name): + if shared.opts.diffusers_offload_mode == "none": + self.to(devices.device, non_blocking=True) + batch_updown = None + batch_ex_bias = None + for net in loaded_networks: + module = net.modules.get(network_layer_name, None) + if module is not None and hasattr(self, 'weight'): + try: + t0 = time.time() + updown, ex_bias = module.calc_updown(weight) + t1 = time.time() + if batch_updown is not None and updown is not None: + batch_updown += updown + else: + batch_updown = updown + if batch_ex_bias is not None and ex_bias is not None: + batch_ex_bias += ex_bias + else: + batch_ex_bias = ex_bias + timer['calc'] += t1 - t0 + if shared.opts.diffusers_offload_mode != "none": + t0 = time.time() + if batch_updown is not None: + batch_updown = batch_updown.to(devices.cpu, non_blocking=True) + if batch_ex_bias is not None: + batch_ex_bias = batch_ex_bias.to(devices.cpu, non_blocking=True) + if devices.backend == "ipex": + # using non_blocking=True here causes NaNs on Intel + torch.xpu.synchronize(devices.device) + t1 = time.time() + timer['move'] += t1 - t0 + except RuntimeError as e: + extra_network_lora.errors[net.name] = extra_network_lora.errors.get(net.name, 0) + 1 + if debug: + module_name = net.modules.get(network_layer_name, None) + shared.log.error(f'LoRA apply weight name="{net.name}" module="{module_name}" layer="{network_layer_name}" {e}') + errors.display(e, 'LoRA') + raise RuntimeError('LoRA apply weight') from e + continue + if module is None: + continue + shared.log.warning(f'LoRA network="{net.name}" layer="{network_layer_name}" unsupported operation') + extra_network_lora.errors[net.name] = extra_network_lora.errors.get(net.name, 0) + 1 + return batch_updown, batch_ex_bias + + +def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv], updown, ex_bias): t0 = time.time() weights_backup = getattr(self, "network_weights_backup", None) bias_backup = getattr(self, "network_bias_backup", None) if weights_backup is None and bias_backup is None: - return + return None, None if weights_backup is not None: + self.weight = None if updown is not None and len(weights_backup.shape) == 4 and weights_backup.shape[1] == 9: # inpainting model. zero pad updown to make channel[1] 4 to 9 updown = torch.nn.functional.pad(updown, (0, 0, 0, 0, 0, 5)) # pylint: disable=not-callable if updown is not None: @@ -312,131 +411,28 @@ def set_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm if getattr(self, "quant_type", None) in ['nf4', 'fp4'] and bnb is not None: self.weight = bnb.nn.Params4bit(new_weight, quant_state=self.quant_state, quant_type=self.quant_type, blocksize=self.blocksize) else: - self.weight.copy_(new_weight, non_blocking=True) + self.weight = torch.nn.Parameter(new_weight, requires_grad=False) del new_weight else: - self.weight.copy_(weights_backup, non_blocking=True) + self.weight = torch.nn.Parameter(weights_backup, requires_grad=False) if hasattr(self, "qweight") and hasattr(self, "freeze"): self.freeze() if bias_backup is not None: + self.bias = None if ex_bias is not None: new_weight = ex_bias.to(devices.device, non_blocking=True) + bias_backup.to(devices.device, non_blocking=True) - self.bias.copy_(new_weight, non_blocking=True) + self.bias = torch.nn.Parameter(new_weight, requires_grad=False) del new_weight else: - self.bias.copy_(bias_backup, non_blocking=True) + self.bias = torch.nn.Parameter(bias_backup, requires_grad=False) else: self.bias = None t1 = time.time() timer['apply'] += t1 - t0 + return self.weight.device, self.weight.dtype -def maybe_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv], wanted_names): # pylint: disable=W0613 - global bnb, backup_size # pylint: disable=W0603 - t0 = time.time() - weights_backup = getattr(self, "network_weights_backup", None) - if weights_backup is None and wanted_names != (): # pylint: disable=C1803 - if getattr(self.weight, "quant_type", None) in ['nf4', 'fp4']: - if bnb is None: - bnb = model_quant.load_bnb('Load network: type=LoRA', silent=True) - if bnb is not None: - with devices.inference_context(): - weights_backup = bnb.functional.dequantize_4bit(self.weight, quant_state=self.weight.quant_state, quant_type=self.weight.quant_type, blocksize=self.weight.blocksize,) - self.quant_state = self.weight.quant_state - self.quant_type = self.weight.quant_type - self.blocksize = self.weight.blocksize - else: - weights_backup = self.weight.clone() - else: - weights_backup = self.weight.clone() - if shared.opts.lora_offload_backup and weights_backup is not None: - weights_backup = weights_backup.to(devices.cpu) - self.network_weights_backup = weights_backup - bias_backup = getattr(self, "network_bias_backup", None) - if bias_backup is None: - if getattr(self, 'bias', None) is not None: - bias_backup = self.bias.clone() - else: - bias_backup = None - if shared.opts.lora_offload_backup and bias_backup is not None: - bias_backup = bias_backup.to(devices.cpu) - self.network_bias_backup = bias_backup - if getattr(self, 'network_weights_backup', None) is not None: - backup_size += self.network_weights_backup.numel() * self.network_weights_backup.element_size() - if getattr(self, 'network_bias_backup', None) is not None: - backup_size += self.network_bias_backup.numel() * self.network_bias_backup.element_size() - t1 = time.time() - timer['backup'] += t1 - t0 - - -def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv]): - """ - Applies the currently selected set of networks to the weights of torch layer self. - If weights already have this particular set of networks applied, does nothing. - If not, restores orginal weights from backup and alters weights according to networks. - """ - network_layer_name = getattr(self, 'network_layer_name', None) - current_names = getattr(self, "network_current_names", ()) - wanted_names = tuple((x.name, x.te_multiplier, x.unet_multiplier, x.dyn_dim) for x in loaded_networks) if len(loaded_networks) > 0 else () - with devices.inference_context(): - if len(loaded_networks) > 0 and network_layer_name is not None and any([net.modules.get(network_layer_name, None) for net in loaded_networks]): # noqa: C419 - maybe_backup_weights(self, wanted_names) - if current_names != wanted_names: - if shared.opts.diffusers_offload_mode == "none": - self.to(devices.device, non_blocking=True) - batch_updown = None - batch_ex_bias = None - for net in loaded_networks: - module = net.modules.get(network_layer_name, None) - if module is not None and hasattr(self, 'weight'): - try: - t0 = time.time() - weight = self.weight.to(devices.device, non_blocking=True) # calculate quant weights once - t1 = time.time() - updown, ex_bias = module.calc_updown(weight) - del weight - t2 = time.time() - timer['move'] += t1 - t0 - timer['calc'] += t2 - t1 - if batch_updown is not None and updown is not None: - batch_updown += updown - else: - batch_updown = updown - if batch_ex_bias is not None and ex_bias is not None: - batch_ex_bias += ex_bias - else: - batch_ex_bias = ex_bias - if shared.opts.diffusers_offload_mode != "none": - t0 = time.time() - if batch_updown is not None: - batch_updown = batch_updown.to(devices.cpu, non_blocking=True) - if batch_ex_bias is not None: - batch_ex_bias = batch_ex_bias.to(devices.cpu, non_blocking=True) - if devices.backend == "ipex": - # using non_blocking=True here causes NaNs on Intel - torch.xpu.synchronize(devices.device) - t1 = time.time() - timer['move'] += t1 - t0 - except RuntimeError as e: - extra_network_lora.errors[net.name] = extra_network_lora.errors.get(net.name, 0) + 1 - if debug: - module_name = net.modules.get(network_layer_name, None) - shared.log.error(f'LoRA apply weight name="{net.name}" module="{module_name}" layer="{network_layer_name}" {e}') - errors.display(e, 'LoRA') - raise RuntimeError('LoRA apply weight') from e - continue - if module is None: - continue - shared.log.warning(f'LoRA network="{net.name}" layer="{network_layer_name}" unsupported operation') - extra_network_lora.errors[net.name] = extra_network_lora.errors.get(net.name, 0) + 1 - self.network_current_names = wanted_names - set_weights(self, batch_updown, batch_ex_bias) # Set or restore weights from backup - if batch_updown is not None or batch_ex_bias is not None: - return self.weight.device - return None - - -def network_load(): +def network_process(): timer['backup'] = 0 timer['calc'] = 0 timer['apply'] = 0 @@ -450,63 +446,39 @@ def network_load(): component = getattr(sd_model, component_name, None) if component is not None and hasattr(component, 'named_modules'): modules += list(component.named_modules()) - devices_used = [] if len(loaded_networks) > 0: pbar = rp.Progress(rp.TextColumn('[cyan]{task.description}'), rp.BarColumn(), rp.TaskProgressColumn(), rp.TimeRemainingColumn(), rp.TimeElapsedColumn(), console=shared.console) task = pbar.add_task(description='Apply network: type=LoRA' , total=len(modules)) else: task = None pbar = nullcontext() - with pbar: + with devices.inference_context(), pbar: + wanted_names = tuple((x.name, x.te_multiplier, x.unet_multiplier, x.dyn_dim) for x in loaded_networks) if len(loaded_networks) > 0 else () + applied = 0 + backup_size = 0 + weights_devices = [] + weights_dtypes = [] for _, module in modules: - if shared.state.interrupted: + network_layer_name = getattr(module, 'network_layer_name', None) + current_names = getattr(module, "network_current_names", ()) + if shared.state.interrupted or network_layer_name is None or current_names == wanted_names: continue - devices_used.append(network_apply_weights(module)) + weight = module.weight.to(devices.device, non_blocking=True) if hasattr(module, 'weight') else None + backup_size += network_backup_weights(module, weight, network_layer_name, wanted_names) + batch_updown, batch_ex_bias = network_calc_weights(module, weight, network_layer_name) + del weight + weights_device, weights_dtype = network_apply_weights(module, batch_updown, batch_ex_bias) + weights_devices.append(weights_device) + weights_dtypes.append(weights_dtype) + module.network_current_names = wanted_names if task is not None: pbar.update(task, advance=1) # progress bar becomes visible if operation takes more than 1sec + if batch_updown is not None or batch_ex_bias is not None: + applied += 1 # pbar.remove_task(task) - if debug: - devices_used = [d for d in devices_used if d is not None] - devices_set = list(set(devices_used)) - shared.log.debug(f'Load network: type=LoRA modules={len(modules)} apply={len(devices_used)} device={devices_set} backup={backup_size} time={get_timers()}') + weights_devices, weights_dtypes = list(set([x for x in weights_devices if x is not None])), list(set([x for x in weights_dtypes if x is not None])) # noqa: C403 + if debug and len(loaded_networks) > 0: + shared.log.debug(f'Load network: type=LoRA modules={len(modules)} networks={len(loaded_networks)} apply={applied} device={weights_devices} dtype={weights_dtypes} backup={backup_size} time={get_timers()}') modules.clear() if shared.opts.diffusers_offload_mode == "sequential": sd_models.set_diffuser_offload(sd_model, op="model") - - -def list_available_networks(): - t0 = time.time() - available_networks.clear() - available_network_aliases.clear() - forbidden_network_aliases.clear() - available_network_hash_lookup.clear() - forbidden_network_aliases.update({"none": 1, "Addams": 1}) - if not os.path.exists(shared.cmd_opts.lora_dir): - shared.log.warning(f'LoRA directory not found: path="{shared.cmd_opts.lora_dir}"') - - def add_network(filename): - if not os.path.isfile(filename): - return - name = os.path.splitext(os.path.basename(filename))[0] - name = name.replace('.', '_') - try: - entry = network.NetworkOnDisk(name, filename) - available_networks[entry.name] = entry - if entry.alias in available_network_aliases: - forbidden_network_aliases[entry.alias.lower()] = 1 - if shared.opts.lora_preferred_name == 'filename': - available_network_aliases[entry.name] = entry - else: - available_network_aliases[entry.alias] = entry - if entry.shorthash: - available_network_hash_lookup[entry.shorthash] = entry - except OSError as e: # should catch FileNotFoundError and PermissionError etc. - shared.log.error(f'LoRA: filename="{filename}" {e}') - - candidates = list(files_cache.list_files(shared.cmd_opts.lora_dir, ext_filter=[".pt", ".ckpt", ".safetensors"])) - with concurrent.futures.ThreadPoolExecutor(max_workers=shared.max_workers) as executor: - for fn in candidates: - executor.submit(add_network, fn) - t1 = time.time() - timer['list'] = t1 - t0 - shared.log.info(f'Available LoRAs: path="{shared.cmd_opts.lora_dir}" items={len(available_networks)} folders={len(forbidden_network_aliases)} time={t1 - t0:.2f}') diff --git a/scripts/xyz_grid.py b/scripts/xyz_grid.py index 60e608c76..0395ce736 100644 --- a/scripts/xyz_grid.py +++ b/scripts/xyz_grid.py @@ -12,6 +12,7 @@ from scripts.xyz_grid_shared import str_permutations, list_to_csv_string, re_range # pylint: disable=no-name-in-module from scripts.xyz_grid_classes import axis_options, AxisOption, SharedSettingsStackHelper # pylint: disable=no-name-in-module from scripts.xyz_grid_draw import draw_xyz_grid # pylint: disable=no-name-in-module +from scripts.xyz_grid_shared import apply_field, apply_task_args, apply_setting, apply_prompt, apply_order, apply_sampler, apply_hr_sampler_name, confirm_samplers, apply_checkpoint, apply_refiner, apply_unet, apply_dict, apply_clip_skip, apply_vae, list_lora, apply_lora, apply_lora_strength, apply_te, apply_styles, apply_upscaler, apply_context, apply_detailer, apply_override, apply_processing, apply_options, apply_seed, format_value_add_label, format_value, format_value_join_list, do_nothing, format_nothing, str_permutations # pylint: disable=no-name-in-module, unused-import from modules import shared, errors, scripts, images, processing from modules.ui_components import ToolButton import modules.ui_symbols as symbols From c57bc1787526fe5c36492ed5dcc6ba5ca833115e Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 1 Dec 2024 15:34:25 -0500 Subject: [PATCH 039/249] balanced offload improvements Signed-off-by: Vladimir Mandic --- modules/call_queue.py | 20 +++++++++------- modules/devices.py | 4 ++-- modules/lora/networks.py | 11 +++++---- modules/memstats.py | 18 +++++++++++--- modules/processing_diffusers.py | 2 ++ modules/prompt_parser_diffusers.py | 38 +++++++++++++++++------------- modules/sd_models.py | 20 +++++++--------- modules/shared.py | 2 +- modules/ui_control.py | 19 +++++++++------ 9 files changed, 80 insertions(+), 54 deletions(-) diff --git a/modules/call_queue.py b/modules/call_queue.py index cdc2fe1f7..11ba7b56e 100644 --- a/modules/call_queue.py +++ b/modules/call_queue.py @@ -73,16 +73,20 @@ def f(*args, extra_outputs_array=extra_outputs, **kwargs): elapsed_m = int(elapsed // 60) elapsed_s = elapsed % 60 elapsed_text = f"{elapsed_m}m {elapsed_s:.2f}s" if elapsed_m > 0 else f"{elapsed_s:.2f}s" - summary = timer.process.summary(min_time=0.1, total=False).replace('=', ' ') - vram_html = '' + summary = timer.process.summary(min_time=0.25, total=False).replace('=', ' ') + gpu = '' + cpu = '' if not shared.mem_mon.disabled: vram = {k: -(v//-(1024*1024)) for k, v in shared.mem_mon.read().items()} - used = round(100 * vram['used'] / (vram['total'] + 0.001)) - if vram.get('active_peak', 0) > 0: - vram_html = " | " - vram_html += f"GPU {max(vram['active_peak'], vram['reserved_peak'])} MB {used}%" - vram_html += f" | retries {vram['retries']} oom {vram['oom']}" if vram.get('retries', 0) > 0 or vram.get('oom', 0) > 0 else '' + peak = max(vram['active_peak'], vram['reserved_peak'], vram['used']) + used = round(100.0 * peak / vram['total']) if vram['total'] > 0 else 0 + if used > 0: + gpu += f"| GPU {peak} MB {used}%" + gpu += f" | retries {vram['retries']} oom {vram['oom']}" if vram.get('retries', 0) > 0 or vram.get('oom', 0) > 0 else '' + ram = shared.ram_stats() + if ram['used'] > 0: + cpu += f"| RAM {ram['used']} GB {round(100.0 * ram['used'] / ram['total'])}%" if isinstance(res, list): - res[-1] += f"

Time: {elapsed_text} | {summary}{vram_html}

" + res[-1] += f"

Time: {elapsed_text} | {summary} {gpu} {cpu}

" return tuple(res) return f diff --git a/modules/devices.py b/modules/devices.py index 64968a30c..71eef5726 100644 --- a/modules/devices.py +++ b/modules/devices.py @@ -224,7 +224,7 @@ def torch_gc(force=False, fast=False): timer.process.records['gc'] = 0 timer.process.records['gc'] += t1 - t0 if not force or collected == 0: - return used_gpu + return used_gpu, used_ram mem = memstats.memory_stats() saved = round(gpu.get('used', 0) - mem.get('gpu', {}).get('used', 0), 2) before = { 'gpu': gpu.get('used', 0), 'ram': ram.get('used', 0) } @@ -233,7 +233,7 @@ def torch_gc(force=False, fast=False): results = { 'collected': collected, 'saved': saved } fn = f'{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access log.debug(f'GC: utilization={utilization} gc={results} before={before} after={after} device={torch.device(get_optimal_device_name())} fn={fn} time={round(t1 - t0, 2)}') # pylint: disable=protected-access - return used_gpu + return used_gpu, used_ram def set_cuda_sync_mode(mode): diff --git a/modules/lora/networks.py b/modules/lora/networks.py index e0f2134c9..21d641af6 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -447,8 +447,8 @@ def network_process(): if component is not None and hasattr(component, 'named_modules'): modules += list(component.named_modules()) if len(loaded_networks) > 0: - pbar = rp.Progress(rp.TextColumn('[cyan]{task.description}'), rp.BarColumn(), rp.TaskProgressColumn(), rp.TimeRemainingColumn(), rp.TimeElapsedColumn(), console=shared.console) - task = pbar.add_task(description='Apply network: type=LoRA' , total=len(modules)) + pbar = rp.Progress(rp.TextColumn('[cyan]Apply network: type=LoRA'), rp.BarColumn(), rp.TaskProgressColumn(), rp.TimeRemainingColumn(), rp.TimeElapsedColumn(), rp.TextColumn('[cyan]{task.description}'), console=shared.console) + task = pbar.add_task(description='' , total=len(modules)) else: task = None pbar = nullcontext() @@ -463,7 +463,8 @@ def network_process(): current_names = getattr(module, "network_current_names", ()) if shared.state.interrupted or network_layer_name is None or current_names == wanted_names: continue - weight = module.weight.to(devices.device, non_blocking=True) if hasattr(module, 'weight') else None + weight = getattr(module, 'weight', None) + weight = weight.to(devices.device, non_blocking=True) if weight is not None else None backup_size += network_backup_weights(module, weight, network_layer_name, wanted_names) batch_updown, batch_ex_bias = network_calc_weights(module, weight, network_layer_name) del weight @@ -472,13 +473,13 @@ def network_process(): weights_dtypes.append(weights_dtype) module.network_current_names = wanted_names if task is not None: - pbar.update(task, advance=1) # progress bar becomes visible if operation takes more than 1sec + pbar.update(task, advance=1, description=f'networks={len(loaded_networks)} modules={len(modules)} apply={applied} backup={backup_size}') if batch_updown is not None or batch_ex_bias is not None: applied += 1 # pbar.remove_task(task) weights_devices, weights_dtypes = list(set([x for x in weights_devices if x is not None])), list(set([x for x in weights_dtypes if x is not None])) # noqa: C403 if debug and len(loaded_networks) > 0: - shared.log.debug(f'Load network: type=LoRA modules={len(modules)} networks={len(loaded_networks)} apply={applied} device={weights_devices} dtype={weights_dtypes} backup={backup_size} time={get_timers()}') + shared.log.debug(f'Load network: type=LoRA networks={len(loaded_networks)} modules={len(modules)} apply={applied} device={weights_devices} dtype={weights_dtypes} backup={backup_size} time={get_timers()}') modules.clear() if shared.opts.diffusers_offload_mode == "sequential": sd_models.set_diffuser_offload(sd_model, op="model") diff --git a/modules/memstats.py b/modules/memstats.py index c417165a2..7836f7636 100644 --- a/modules/memstats.py +++ b/modules/memstats.py @@ -5,11 +5,12 @@ fail_once = False +def gb(val: float): + return round(val / 1024 / 1024 / 1024, 2) + + def memory_stats(): global fail_once # pylint: disable=global-statement - def gb(val: float): - return round(val / 1024 / 1024 / 1024, 2) - mem = {} try: process = psutil.Process(os.getpid()) @@ -38,3 +39,14 @@ def gb(val: float): except Exception: pass return mem + + +def ram_stats(): + try: + process = psutil.Process(os.getpid()) + res = process.memory_info() + ram_total = 100 * res.rss / process.memory_percent() + ram = { 'used': gb(res.rss), 'total': gb(ram_total) } + return ram + except Exception: + return { 'used': 0, 'total': 0 } diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 463a15280..c605a761c 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -83,6 +83,8 @@ def process_base(p: processing.StableDiffusionProcessing): try: t0 = time.time() sd_models_compile.check_deepcache(enable=True) + if shared.opts.diffusers_offload_mode == "balanced": + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) sd_models.move_model(shared.sd_model, devices.device) if hasattr(shared.sd_model, 'unet'): sd_models.move_model(shared.sd_model.unet, devices.device) diff --git a/modules/prompt_parser_diffusers.py b/modules/prompt_parser_diffusers.py index c74731c6d..d2093351a 100644 --- a/modules/prompt_parser_diffusers.py +++ b/modules/prompt_parser_diffusers.py @@ -39,8 +39,8 @@ def prepare_model(pipe = None): pipe = pipe.pipe if not hasattr(pipe, "text_encoder"): return None - if shared.opts.diffusers_offload_mode == "balanced": - pipe = sd_models.apply_balanced_offload(pipe) + # if shared.opts.diffusers_offload_mode == "balanced": + # pipe = sd_models.apply_balanced_offload(pipe) elif hasattr(pipe, "maybe_free_model_hooks"): pipe.maybe_free_model_hooks() devices.torch_gc() @@ -79,8 +79,8 @@ def __init__(self, prompts, negative_prompts, steps, clip_skip, p): self.scheduled_encode(pipe, batchidx) else: self.encode(pipe, prompt, negative_prompt, batchidx) - if shared.opts.diffusers_offload_mode == "balanced": - pipe = sd_models.apply_balanced_offload(pipe) + # if shared.opts.diffusers_offload_mode == "balanced": + # pipe = sd_models.apply_balanced_offload(pipe) self.checkcache(p) debug(f"Prompt encode: time={(time.time() - t0):.3f}") @@ -199,8 +199,6 @@ def __call__(self, key, step=0): def compel_hijack(self, token_ids: torch.Tensor, attention_mask: typing.Optional[torch.Tensor] = None) -> torch.Tensor: - if not devices.same_device(self.text_encoder.device, devices.device): - sd_models.move_model(self.text_encoder, devices.device) needs_hidden_states = self.returned_embeddings_type != 1 text_encoder_output = self.text_encoder(token_ids, attention_mask, output_hidden_states=needs_hidden_states, return_dict=True) @@ -377,25 +375,31 @@ def prepare_embedding_providers(pipe, clip_skip) -> list[EmbeddingsProvider]: embedding_type = -(clip_skip + 1) else: embedding_type = clip_skip + embedding_args = { + 'truncate': False, + 'returned_embeddings_type': embedding_type, + 'device': device, + 'dtype_for_device_getter': lambda device: devices.dtype, + } if getattr(pipe, "prior_pipe", None) is not None and getattr(pipe.prior_pipe, "tokenizer", None) is not None and getattr(pipe.prior_pipe, "text_encoder", None) is not None: - provider = EmbeddingsProvider(padding_attention_mask_value=0, tokenizer=pipe.prior_pipe.tokenizer, text_encoder=pipe.prior_pipe.text_encoder, truncate=False, returned_embeddings_type=embedding_type, device=device) + provider = EmbeddingsProvider(padding_attention_mask_value=0, tokenizer=pipe.prior_pipe.tokenizer, text_encoder=pipe.prior_pipe.text_encoder, **embedding_args) embeddings_providers.append(provider) - no_mask_provider = EmbeddingsProvider(padding_attention_mask_value=1 if "sote" in pipe.sd_checkpoint_info.name.lower() else 0, tokenizer=pipe.prior_pipe.tokenizer, text_encoder=pipe.prior_pipe.text_encoder, truncate=False, returned_embeddings_type=embedding_type, device=device) + no_mask_provider = EmbeddingsProvider(padding_attention_mask_value=1 if "sote" in pipe.sd_checkpoint_info.name.lower() else 0, tokenizer=pipe.prior_pipe.tokenizer, text_encoder=pipe.prior_pipe.text_encoder, **embedding_args) embeddings_providers.append(no_mask_provider) elif getattr(pipe, "tokenizer", None) is not None and getattr(pipe, "text_encoder", None) is not None: - if not devices.same_device(pipe.text_encoder.device, devices.device): - sd_models.move_model(pipe.text_encoder, devices.device) - provider = EmbeddingsProvider(tokenizer=pipe.tokenizer, text_encoder=pipe.text_encoder, truncate=False, returned_embeddings_type=embedding_type, device=device) + if pipe.text_encoder.__class__.__name__.startswith('CLIP'): + sd_models.move_model(pipe.text_encoder, devices.device, force=True) + provider = EmbeddingsProvider(tokenizer=pipe.tokenizer, text_encoder=pipe.text_encoder, **embedding_args) embeddings_providers.append(provider) if getattr(pipe, "tokenizer_2", None) is not None and getattr(pipe, "text_encoder_2", None) is not None: - if not devices.same_device(pipe.text_encoder_2.device, devices.device): - sd_models.move_model(pipe.text_encoder_2, devices.device) - provider = EmbeddingsProvider(tokenizer=pipe.tokenizer_2, text_encoder=pipe.text_encoder_2, truncate=False, returned_embeddings_type=embedding_type, device=device) + if pipe.text_encoder_2.__class__.__name__.startswith('CLIP'): + sd_models.move_model(pipe.text_encoder_2, devices.device, force=True) + provider = EmbeddingsProvider(tokenizer=pipe.tokenizer_2, text_encoder=pipe.text_encoder_2, **embedding_args) embeddings_providers.append(provider) if getattr(pipe, "tokenizer_3", None) is not None and getattr(pipe, "text_encoder_3", None) is not None: - if not devices.same_device(pipe.text_encoder_3.device, devices.device): - sd_models.move_model(pipe.text_encoder_3, devices.device) - provider = EmbeddingsProvider(tokenizer=pipe.tokenizer_3, text_encoder=pipe.text_encoder_3, truncate=False, returned_embeddings_type=embedding_type, device=device) + if pipe.text_encoder_3.__class__.__name__.startswith('CLIP'): + sd_models.move_model(pipe.text_encoder_3, devices.device, force=True) + provider = EmbeddingsProvider(tokenizer=pipe.tokenizer_3, text_encoder=pipe.text_encoder_3, **embedding_args) embeddings_providers.append(provider) return embeddings_providers diff --git a/modules/sd_models.py b/modules/sd_models.py index 24ceff5ee..83bf6f994 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -401,7 +401,6 @@ def pre_forward(self, module, *args, **kwargs): module._hf_hook.execution_device = torch.device(devices.device) # pylint: disable=protected-access module.balanced_offload_device_map = device_map module.balanced_offload_max_memory = max_memory - module.balanced_offload_active = True return args, kwargs def post_forward(self, module, output): @@ -429,7 +428,7 @@ def apply_balanced_offload(sd_model): checkpoint_name = sd_model.__class__.__name__ def apply_balanced_offload_to_module(pipe): - used_gpu = devices.torch_gc(fast=True) + used_gpu, used_ram = devices.torch_gc(fast=True) if hasattr(pipe, "pipe"): apply_balanced_offload_to_module(pipe.pipe) if hasattr(pipe, "_internal_dict"): @@ -438,20 +437,21 @@ def apply_balanced_offload_to_module(pipe): keys = get_signature(pipe).keys() for module_name in keys: # pylint: disable=protected-access module = getattr(pipe, module_name, None) - balanced_offload_active = getattr(module, "balanced_offload_active", None) - if isinstance(module, torch.nn.Module) and (balanced_offload_active is None or balanced_offload_active): + if isinstance(module, torch.nn.Module): network_layer_name = getattr(module, "network_layer_name", None) device_map = getattr(module, "balanced_offload_device_map", None) max_memory = getattr(module, "balanced_offload_max_memory", None) module = accelerate.hooks.remove_hook_from_module(module, recurse=True) try: if used_gpu > 100 * shared.opts.diffusers_offload_min_gpu_memory: + debug_move(f'Balanced offload: gpu={used_gpu} ram={used_ram} current={module.device} target={devices.cpu} component={module.__class__.__name__}') module = module.to(devices.cpu, non_blocking=True) - used_gpu = devices.torch_gc(fast=True) + used_gpu, used_ram = devices.torch_gc(fast=True) + else: + debug_move(f'Balanced offload: gpu={used_gpu} ram={used_ram} current={module.device} target={devices.cpu} component={module.__class__.__name__}') module.offload_dir = os.path.join(shared.opts.accelerate_offload_path, checkpoint_name, module_name) module = accelerate.hooks.add_hook_to_module(module, offload_hook_instance, append=True) module._hf_hook.execution_device = torch.device(devices.device) # pylint: disable=protected-access - module.balanced_offload_active = False if network_layer_name: module.network_layer_name = network_layer_name if device_map and max_memory: @@ -515,13 +515,13 @@ def move_model(model, device=None, force=False): shared.log.error(f'Model move execution device: device={device} {e}') if getattr(model, 'has_accelerate', False) and not force: return - if hasattr(model, "device") and devices.normalize_device(model.device) == devices.normalize_device(device): + if hasattr(model, "device") and devices.normalize_device(model.device) == devices.normalize_device(device) and not force: return try: t0 = time.time() try: if hasattr(model, 'to'): - model.to(device) + model.to(device, non_blocking=True) if hasattr(model, "prior_pipe"): model.prior_pipe.to(device) except Exception as e0: @@ -551,7 +551,7 @@ def move_model(model, device=None, force=False): if 'move' not in process_timer.records: process_timer.records['move'] = 0 process_timer.records['move'] += t1 - t0 - if os.environ.get('SD_MOVE_DEBUG', None) or (t1-t0) > 1: + if os.environ.get('SD_MOVE_DEBUG', None) or (t1-t0) > 2: shared.log.debug(f'Model move: device={device} class={model.__class__.__name__} accelerate={getattr(model, "has_accelerate", False)} fn={fn} time={t1-t0:.2f}') # pylint: disable=protected-access devices.torch_gc() @@ -1492,8 +1492,6 @@ def disable_offload(sd_model): module = getattr(sd_model, module_name, None) if isinstance(module, torch.nn.Module): network_layer_name = getattr(module, "network_layer_name", None) - if getattr(module, "balanced_offload_active", None) is not None: - module.balanced_offload_active = None module = remove_hook_from_module(module, recurse=True) if network_layer_name: module.network_layer_name = network_layer_name diff --git a/modules/shared.py b/modules/shared.py index b8fac60ab..07599a28a 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -20,7 +20,7 @@ from modules.paths import models_path, script_path, data_path, sd_configs_path, sd_default_config, sd_model_file, default_sd_model_file, extensions_dir, extensions_builtin_dir # pylint: disable=W0611 from modules.dml import memory_providers, default_memory_provider, directml_do_hijack from modules.onnx_impl import initialize_onnx, execution_providers -from modules.memstats import memory_stats +from modules.memstats import memory_stats, ram_stats from modules.ui_components import DropdownEditable import modules.interrogate import modules.memmon diff --git a/modules/ui_control.py b/modules/ui_control.py index 072d9b9c9..59db12fc5 100644 --- a/modules/ui_control.py +++ b/modules/ui_control.py @@ -29,15 +29,20 @@ def return_stats(t: float = None): elapsed_m = int(elapsed // 60) elapsed_s = elapsed % 60 elapsed_text = f"Time: {elapsed_m}m {elapsed_s:.2f}s |" if elapsed_m > 0 else f"Time: {elapsed_s:.2f}s |" - summary = timer.process.summary(min_time=0.1, total=False).replace('=', ' ') - vram_html = '' + summary = timer.process.summary(min_time=0.25, total=False).replace('=', ' ') + gpu = '' + cpu = '' if not shared.mem_mon.disabled: vram = {k: -(v//-(1024*1024)) for k, v in shared.mem_mon.read().items()} - used = round(100 * vram['used'] / (vram['total'] + 0.001)) - if vram.get('active_peak', 0) > 0: - vram_html += f"| GPU {max(vram['active_peak'], vram['reserved_peak'])} MB {used}%" - vram_html += f" | retries {vram['retries']} oom {vram['oom']}" if vram.get('retries', 0) > 0 or vram.get('oom', 0) > 0 else '' - return f"

{elapsed_text} {summary} {vram_html}

" + peak = max(vram['active_peak'], vram['reserved_peak'], vram['used']) + used = round(100.0 * peak / vram['total']) if vram['total'] > 0 else 0 + if used > 0: + gpu += f"| GPU {peak} MB {used}%" + gpu += f" | retries {vram['retries']} oom {vram['oom']}" if vram.get('retries', 0) > 0 or vram.get('oom', 0) > 0 else '' + ram = shared.ram_stats() + if ram['used'] > 0: + cpu += f"| RAM {ram['used']} GB {round(100.0 * ram['used'] / ram['total'])}%" + return f"

Time: {elapsed_text} | {summary} {gpu} {cpu}

" def return_controls(res, t: float = None): From 56e234cef5d776852be0f3650b5347f5b0102cd2 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Mon, 2 Dec 2024 00:29:01 +0300 Subject: [PATCH 040/249] Reduce balanced offload max gpu memory to 0.70 --- modules/shared.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/shared.py b/modules/shared.py index 07599a28a..54133cb1e 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -561,7 +561,7 @@ def get_default_modes(): "diffusers_generator_device": OptionInfo("GPU", "Generator device", gr.Radio, {"choices": ["GPU", "CPU", "Unset"]}), "diffusers_offload_mode": OptionInfo(startup_offload_mode, "Model offload mode", gr.Radio, {"choices": ['none', 'balanced', 'model', 'sequential']}), "diffusers_offload_min_gpu_memory": OptionInfo(0.25, "Balanced offload GPU low watermark", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01 }), - "diffusers_offload_max_gpu_memory": OptionInfo(0.75, "Balanced offload GPU high watermark", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01 }), + "diffusers_offload_max_gpu_memory": OptionInfo(0.70, "Balanced offload GPU high watermark", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01 }), "diffusers_offload_max_cpu_memory": OptionInfo(0.75, "Balanced offload CPU high watermark", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01 }), "diffusers_vae_upcast": OptionInfo("default", "VAE upcasting", gr.Radio, {"choices": ['default', 'true', 'false']}), "diffusers_vae_slicing": OptionInfo(True, "VAE slicing"), From 2ef23bb1a5767761e09faa9274096ac911c8d923 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 1 Dec 2024 17:13:53 -0500 Subject: [PATCH 041/249] memory optimizations Signed-off-by: Vladimir Mandic --- modules/processing.py | 1 + modules/processing_args.py | 24 ++++++++++++++++-------- modules/sd_models.py | 29 +++++++++++++++-------------- 3 files changed, 32 insertions(+), 22 deletions(-) diff --git a/modules/processing.py b/modules/processing.py index ebbaf7272..095eba54c 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -483,4 +483,5 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: for stat in stats[:20]: frame = stat.traceback[0] shared.log.debug(f' file="{frame.filename}":{frame.lineno} size={stat.size}') + devices.torch_gc(force=True) return processed diff --git a/modules/processing_args.py b/modules/processing_args.py index a716b685e..4ce552825 100644 --- a/modules/processing_args.py +++ b/modules/processing_args.py @@ -12,7 +12,8 @@ from modules.api import helpers -debug = shared.log.trace if os.environ.get('SD_DIFFUSERS_DEBUG', None) is not None else lambda *args, **kwargs: None +debug_enabled = os.environ.get('SD_DIFFUSERS_DEBUG', None) +debug_log = shared.log.trace if os.environ.get('SD_DIFFUSERS_DEBUG', None) is not None else lambda *args, **kwargs: None def task_specific_kwargs(p, model): @@ -93,7 +94,8 @@ def task_specific_kwargs(p, model): 'target_subject_category': getattr(p, 'prompt', '').split()[-1], 'output_type': 'pil', } - debug(f'Diffusers task specific args: {task_args}') + if debug_enabled: + debug_log(f'Diffusers task specific args: {task_args}') return task_args @@ -108,7 +110,8 @@ def set_pipeline_args(p, model, prompts: list, negative_prompts: list, prompts_2 signature = inspect.signature(type(model).__call__, follow_wrapped=True) possible = list(signature.parameters) - debug(f'Diffusers pipeline possible: {possible}') + if debug_enabled: + debug_log(f'Diffusers pipeline possible: {possible}') prompts, negative_prompts, prompts_2, negative_prompts_2 = fix_prompts(prompts, negative_prompts, prompts_2, negative_prompts_2) steps = kwargs.get("num_inference_steps", None) or len(getattr(p, 'timesteps', ['1'])) clip_skip = kwargs.pop("clip_skip", 1) @@ -159,6 +162,8 @@ def set_pipeline_args(p, model, prompts: list, negative_prompts: list, prompts_2 args['negative_prompt'] = negative_prompts[0] else: args['negative_prompt'] = negative_prompts + if prompt_parser_diffusers.embedder is not None and not prompt_parser_diffusers.embedder.scheduled_prompt: # not scheduled so we dont need it anymore + prompt_parser_diffusers.embedder = None if 'clip_skip' in possible and parser == 'fixed': if clip_skip == 1: @@ -248,14 +253,16 @@ def set_pipeline_args(p, model, prompts: list, negative_prompts: list, prompts_2 if arg in possible: args[arg] = task_kwargs[arg] task_args = getattr(p, 'task_args', {}) - debug(f'Diffusers task args: {task_args}') + if debug_enabled: + debug_log(f'Diffusers task args: {task_args}') for k, v in task_args.items(): if k in possible: args[k] = v else: - debug(f'Diffusers unknown task args: {k}={v}') + debug_log(f'Diffusers unknown task args: {k}={v}') cross_attention_args = getattr(p, 'cross_attention_kwargs', {}) - debug(f'Diffusers cross-attention args: {cross_attention_args}') + if debug_enabled: + debug_log(f'Diffusers cross-attention args: {cross_attention_args}') for k, v in cross_attention_args.items(): if args.get('cross_attention_kwargs', None) is None: args['cross_attention_kwargs'] = {} @@ -273,7 +280,7 @@ def set_pipeline_args(p, model, prompts: list, negative_prompts: list, prompts_2 # handle implicit controlnet if 'control_image' in possible and 'control_image' not in args and 'image' in args: - debug('Diffusers: set control image') + debug_log('Diffusers: set control image') args['control_image'] = args['image'] sd_hijack_hypertile.hypertile_set(p, hr=len(getattr(p, 'init_images', [])) > 0) @@ -309,5 +316,6 @@ def set_pipeline_args(p, model, prompts: list, negative_prompts: list, prompts_2 if shared.cmd_opts.profile: t1 = time.time() shared.log.debug(f'Profile: pipeline args: {t1-t0:.2f}') - debug(f'Diffusers pipeline args: {args}') + if debug_enabled: + debug_log(f'Diffusers pipeline args: {args}') return args diff --git a/modules/sd_models.py b/modules/sd_models.py index 83bf6f994..37567962c 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -443,23 +443,24 @@ def apply_balanced_offload_to_module(pipe): max_memory = getattr(module, "balanced_offload_max_memory", None) module = accelerate.hooks.remove_hook_from_module(module, recurse=True) try: - if used_gpu > 100 * shared.opts.diffusers_offload_min_gpu_memory: - debug_move(f'Balanced offload: gpu={used_gpu} ram={used_ram} current={module.device} target={devices.cpu} component={module.__class__.__name__}') - module = module.to(devices.cpu, non_blocking=True) - used_gpu, used_ram = devices.torch_gc(fast=True) - else: - debug_move(f'Balanced offload: gpu={used_gpu} ram={used_ram} current={module.device} target={devices.cpu} component={module.__class__.__name__}') - module.offload_dir = os.path.join(shared.opts.accelerate_offload_path, checkpoint_name, module_name) - module = accelerate.hooks.add_hook_to_module(module, offload_hook_instance, append=True) - module._hf_hook.execution_device = torch.device(devices.device) # pylint: disable=protected-access - if network_layer_name: - module.network_layer_name = network_layer_name - if device_map and max_memory: - module.balanced_offload_device_map = device_map - module.balanced_offload_max_memory = max_memory + do_offload = used_gpu > 100 * shared.opts.diffusers_offload_min_gpu_memory + debug_move(f'Balanced offload: gpu={used_gpu} ram={used_ram} current={module.device} dtype={module.dtype} op={"move" if do_offload else "skip"} component={module.__class__.__name__}') + if do_offload: + module = module.to(devices.cpu) + used_gpu, used_ram = devices.torch_gc(fast=True, force=True) except Exception as e: if 'bitsandbytes' not in str(e): shared.log.error(f'Balanced offload: module={module_name} {e}') + if os.environ.get('SD_MOVE_DEBUG', None): + errors.display(e, f'Balanced offload: module={module_name}') + module.offload_dir = os.path.join(shared.opts.accelerate_offload_path, checkpoint_name, module_name) + module = accelerate.hooks.add_hook_to_module(module, offload_hook_instance, append=True) + module._hf_hook.execution_device = torch.device(devices.device) # pylint: disable=protected-access + if network_layer_name: + module.network_layer_name = network_layer_name + if device_map and max_memory: + module.balanced_offload_device_map = device_map + module.balanced_offload_max_memory = max_memory apply_balanced_offload_to_module(sd_model) if hasattr(sd_model, "pipe"): From 318bbba50fa1f2e8c0820d1af15ce81c9b45caf1 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 1 Dec 2024 17:26:34 -0500 Subject: [PATCH 042/249] cleanup Signed-off-by: Vladimir Mandic --- modules/sd_checkpoint.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/sd_checkpoint.py b/modules/sd_checkpoint.py index 2f6533ef0..6ab396329 100644 --- a/modules/sd_checkpoint.py +++ b/modules/sd_checkpoint.py @@ -253,6 +253,8 @@ def select_checkpoint(op='model'): model_checkpoint = shared.opts.data.get('sd_model_refiner', None) else: model_checkpoint = shared.opts.sd_model_checkpoint + if len(model_checkpoint) < 3: + return None if model_checkpoint is None or model_checkpoint == 'None': return None checkpoint_info = get_closet_checkpoint_match(model_checkpoint) From 8bc2c66d73318d51523b2f8543ba0e75774fce06 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 2 Dec 2024 10:51:41 -0500 Subject: [PATCH 043/249] lora add fuse Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 5 +- extensions-builtin/Lora/network_overrides.py | 1 - modules/lora/extra_networks_lora.py | 3 +- modules/lora/networks.py | 61 ++++++++++++++------ modules/shared.py | 10 ++-- scripts/xyz_grid.py | 2 +- 6 files changed, 54 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c62b6b917..a9109341d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Change Log for SD.Next -## Update for 2024-11-30 +## Update for 2024-12-02 ### New models and integrations @@ -32,6 +32,9 @@ ### UI and workflow improvements +- **LoRA** handler rewrite + - LoRA weights are no longer calculated on-the-fly during model execution, but are pre-calculated at the start + this results in perceived overhead on generate startup, but results in overall faster execution as LoRA does not need to be processed on each step - **Model loader** improvements: - detect model components on model load fail - allow passing absolute path to model loader diff --git a/extensions-builtin/Lora/network_overrides.py b/extensions-builtin/Lora/network_overrides.py index 5334f3c1b..b5c28b718 100644 --- a/extensions-builtin/Lora/network_overrides.py +++ b/extensions-builtin/Lora/network_overrides.py @@ -26,7 +26,6 @@ force_models = [ # forced always 'sc', - # 'sd3', 'kandinsky', 'hunyuandit', 'auraflow', diff --git a/modules/lora/extra_networks_lora.py b/modules/lora/extra_networks_lora.py index d58cebd8f..57966550a 100644 --- a/modules/lora/extra_networks_lora.py +++ b/modules/lora/extra_networks_lora.py @@ -124,7 +124,7 @@ def activate(self, p, params_list, step=0): self.model = shared.opts.sd_model_checkpoint names, te_multipliers, unet_multipliers, dyn_dims = parse(p, params_list, step) networks.network_load(names, te_multipliers, unet_multipliers, dyn_dims) # load - networks.network_process() + networks.network_activate() if len(networks.loaded_networks) > 0 and step == 0: infotext(p) prompt(p) @@ -141,6 +141,7 @@ def deactivate(self, p): shared.sd_model.unload_lora_weights() # fails for non-CLIP models except Exception: pass + networks.network_deactivate() t1 = time.time() networks.timer['restore'] += t1 - t0 if self.active and networks.debug: diff --git a/modules/lora/networks.py b/modules/lora/networks.py index 21d641af6..48073774c 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -310,12 +310,15 @@ def network_load(names, te_multipliers=None, unet_multipliers=None, dyn_dims=Non def network_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv], weight, network_layer_name, wanted_names): global bnb # pylint: disable=W0603 backup_size = 0 - if len(loaded_networks) > 0 and network_layer_name is not None and any([net.modules.get(network_layer_name, None) for net in loaded_networks]): # noqa: C419 + if len(loaded_networks) > 0 and network_layer_name is not None and any([net.modules.get(network_layer_name, None) for net in loaded_networks]): # noqa: C419 # pylint: disable=R1729 t0 = time.time() + weights_backup = getattr(self, "network_weights_backup", None) if weights_backup is None and wanted_names != (): # pylint: disable=C1803 self.network_weights_backup = None - if getattr(weight, "quant_type", None) in ['nf4', 'fp4']: + if shared.opts.lora_fuse_diffusers: + weights_backup = True + elif getattr(weight, "quant_type", None) in ['nf4', 'fp4']: if bnb is None: bnb = model_quant.load_bnb('Load network: type=LoRA', silent=True) if bnb is not None: @@ -328,22 +331,26 @@ def network_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.n weights_backup = weight.clone() else: weights_backup = weight.clone() - if shared.opts.lora_offload_backup and weights_backup is not None: + if shared.opts.lora_offload_backup and weights_backup is not None and isinstance(weights_backup, torch.Tensor): weights_backup = weights_backup.to(devices.cpu) self.network_weights_backup = weights_backup bias_backup = getattr(self, "network_bias_backup", None) if bias_backup is None: if getattr(self, 'bias', None) is not None: - bias_backup = self.bias.clone() + if shared.opts.lora_fuse_diffusers: + bias_backup = True + else: + bias_backup = self.bias.clone() else: bias_backup = None - if shared.opts.lora_offload_backup and bias_backup is not None: + if shared.opts.lora_offload_backup and bias_backup is not None and isinstance(bias_backup, torch.Tensor): bias_backup = bias_backup.to(devices.cpu) self.network_bias_backup = bias_backup + if getattr(self, 'network_weights_backup', None) is not None: - backup_size += self.network_weights_backup.numel() * self.network_weights_backup.element_size() + backup_size += self.network_weights_backup.numel() * self.network_weights_backup.element_size() if isinstance(self.network_weights_backup, torch.Tensor) else 0 if getattr(self, 'network_bias_backup', None) is not None: - backup_size += self.network_bias_backup.numel() * self.network_bias_backup.element_size() + backup_size += self.network_bias_backup.numel() * self.network_bias_backup.element_size() if isinstance(self.network_bias_backup, torch.Tensor) else 0 t1 = time.time() timer['backup'] += t1 - t0 return backup_size @@ -396,18 +403,24 @@ def network_calc_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn. return batch_updown, batch_ex_bias -def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv], updown, ex_bias): +def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv], updown, ex_bias, apply: bool = True): t0 = time.time() weights_backup = getattr(self, "network_weights_backup", None) bias_backup = getattr(self, "network_bias_backup", None) if weights_backup is None and bias_backup is None: return None, None if weights_backup is not None: - self.weight = None + if isinstance(weights_backup, bool): + weights_backup = self.weight + else: + self.weight = None if updown is not None and len(weights_backup.shape) == 4 and weights_backup.shape[1] == 9: # inpainting model. zero pad updown to make channel[1] 4 to 9 updown = torch.nn.functional.pad(updown, (0, 0, 0, 0, 0, 5)) # pylint: disable=not-callable if updown is not None: - new_weight = updown.to(devices.device, non_blocking=True) + weights_backup.to(devices.device, non_blocking=True) + if apply: + new_weight = weights_backup.to(devices.device, non_blocking=True) + updown.to(devices.device, non_blocking=True) + else: + new_weight = weights_backup.to(devices.device, non_blocking=True) - updown.to(devices.device, non_blocking=True) if getattr(self, "quant_type", None) in ['nf4', 'fp4'] and bnb is not None: self.weight = bnb.nn.Params4bit(new_weight, quant_state=self.quant_state, quant_type=self.quant_type, blocksize=self.blocksize) else: @@ -418,9 +431,15 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn if hasattr(self, "qweight") and hasattr(self, "freeze"): self.freeze() if bias_backup is not None: - self.bias = None + if isinstance(bias_backup, bool): + bias_backup = self.bias + else: + self.bias = None if ex_bias is not None: - new_weight = ex_bias.to(devices.device, non_blocking=True) + bias_backup.to(devices.device, non_blocking=True) + if apply: + new_weight = bias_backup.to(devices.device, non_blocking=True) + ex_bias.to(devices.device, non_blocking=True) + else: + new_weight = bias_backup.to(devices.device, non_blocking=True) - ex_bias.to(devices.device, non_blocking=True) self.bias = torch.nn.Parameter(new_weight, requires_grad=False) del new_weight else: @@ -432,7 +451,10 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn return self.weight.device, self.weight.dtype -def network_process(): +def network_deactivate(): + pass + +def network_activate(): timer['backup'] = 0 timer['calc'] = 0 timer['apply'] = 0 @@ -462,24 +484,25 @@ def network_process(): network_layer_name = getattr(module, 'network_layer_name', None) current_names = getattr(module, "network_current_names", ()) if shared.state.interrupted or network_layer_name is None or current_names == wanted_names: + if task is not None: + pbar.update(task, advance=1, description=f'networks={len(loaded_networks)} skip') continue weight = getattr(module, 'weight', None) weight = weight.to(devices.device, non_blocking=True) if weight is not None else None backup_size += network_backup_weights(module, weight, network_layer_name, wanted_names) batch_updown, batch_ex_bias = network_calc_weights(module, weight, network_layer_name) - del weight weights_device, weights_dtype = network_apply_weights(module, batch_updown, batch_ex_bias) weights_devices.append(weights_device) weights_dtypes.append(weights_dtype) + if batch_updown is not None or batch_ex_bias is not None: + applied += 1 + del weight, batch_updown, batch_ex_bias module.network_current_names = wanted_names if task is not None: pbar.update(task, advance=1, description=f'networks={len(loaded_networks)} modules={len(modules)} apply={applied} backup={backup_size}') - if batch_updown is not None or batch_ex_bias is not None: - applied += 1 - # pbar.remove_task(task) - weights_devices, weights_dtypes = list(set([x for x in weights_devices if x is not None])), list(set([x for x in weights_dtypes if x is not None])) # noqa: C403 + weights_devices, weights_dtypes = list(set([x for x in weights_devices if x is not None])), list(set([x for x in weights_dtypes if x is not None])) # noqa: C403 # pylint: disable=R1718 if debug and len(loaded_networks) > 0: - shared.log.debug(f'Load network: type=LoRA networks={len(loaded_networks)} modules={len(modules)} apply={applied} device={weights_devices} dtype={weights_dtypes} backup={backup_size} time={get_timers()}') + shared.log.debug(f'Load network: type=LoRA networks={len(loaded_networks)} modules={len(modules)} apply={applied} device={weights_devices} dtype={weights_dtypes} backup={backup_size} fuse={shared.opts.lora_fuse_diffusers} time={get_timers()}') modules.clear() if shared.opts.diffusers_offload_mode == "sequential": sd_models.set_diffuser_offload(sd_model, op="model") diff --git a/modules/shared.py b/modules/shared.py index 54133cb1e..8f809fa8b 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -20,7 +20,7 @@ from modules.paths import models_path, script_path, data_path, sd_configs_path, sd_default_config, sd_model_file, default_sd_model_file, extensions_dir, extensions_builtin_dir # pylint: disable=W0611 from modules.dml import memory_providers, default_memory_provider, directml_do_hijack from modules.onnx_impl import initialize_onnx, execution_providers -from modules.memstats import memory_stats, ram_stats +from modules.memstats import memory_stats from modules.ui_components import DropdownEditable import modules.interrogate import modules.memmon @@ -903,16 +903,16 @@ def get_default_modes(): "wildcards_enabled": OptionInfo(True, "Enable file wildcards support"), "extra_networks_lora_sep": OptionInfo("

LoRA

", "", gr.HTML), "extra_networks_default_multiplier": OptionInfo(1.0, "Default strength", gr.Slider, {"minimum": 0.0, "maximum": 2.0, "step": 0.01}), - "lora_preferred_name": OptionInfo("filename", "LoRA preferred name", gr.Radio, {"choices": ["filename", "alias"]}), + "lora_preferred_name": OptionInfo("filename", "LoRA preferred name", gr.Radio, {"choices": ["filename", "alias"], "visible": False}), "lora_add_hashes_to_infotext": OptionInfo(False, "LoRA add hash info"), + "lora_fuse_diffusers": OptionInfo(False if not cmd_opts.use_openvino else True, "LoRA fuse directly to model"), + "lora_load_gpu": OptionInfo(True if not (cmd_opts.lowvram or cmd_opts.medvram) else False, "LoRA load directly to GPU"), + "lora_offload_backup": OptionInfo(True, "LoRA offload backup weights"), "lora_force_diffusers": OptionInfo(False if not cmd_opts.use_openvino else True, "LoRA force loading of all models using Diffusers"), "lora_maybe_diffusers": OptionInfo(False, "LoRA force loading of specific models using Diffusers"), - "lora_fuse_diffusers": OptionInfo(False if not cmd_opts.use_openvino else True, "LoRA use fuse when possible"), "lora_apply_tags": OptionInfo(0, "LoRA auto-apply tags", gr.Slider, {"minimum": -1, "maximum": 32, "step": 1}), "lora_in_memory_limit": OptionInfo(0, "LoRA memory cache", gr.Slider, {"minimum": 0, "maximum": 24, "step": 1}), "lora_quant": OptionInfo("NF4","LoRA precision in quantized models", gr.Radio, {"choices": ["NF4", "FP4"]}), - "lora_load_gpu": OptionInfo(True if not (cmd_opts.lowvram or cmd_opts.medvram) else False, "Load LoRA directly to GPU"), - "lora_offload_backup": OptionInfo(True, "Offload LoRA Backup Weights"), })) options_templates.update(options_section((None, "Internal options"), { diff --git a/scripts/xyz_grid.py b/scripts/xyz_grid.py index 0395ce736..bb067ea21 100644 --- a/scripts/xyz_grid.py +++ b/scripts/xyz_grid.py @@ -12,7 +12,7 @@ from scripts.xyz_grid_shared import str_permutations, list_to_csv_string, re_range # pylint: disable=no-name-in-module from scripts.xyz_grid_classes import axis_options, AxisOption, SharedSettingsStackHelper # pylint: disable=no-name-in-module from scripts.xyz_grid_draw import draw_xyz_grid # pylint: disable=no-name-in-module -from scripts.xyz_grid_shared import apply_field, apply_task_args, apply_setting, apply_prompt, apply_order, apply_sampler, apply_hr_sampler_name, confirm_samplers, apply_checkpoint, apply_refiner, apply_unet, apply_dict, apply_clip_skip, apply_vae, list_lora, apply_lora, apply_lora_strength, apply_te, apply_styles, apply_upscaler, apply_context, apply_detailer, apply_override, apply_processing, apply_options, apply_seed, format_value_add_label, format_value, format_value_join_list, do_nothing, format_nothing, str_permutations # pylint: disable=no-name-in-module, unused-import +from scripts.xyz_grid_shared import apply_field, apply_task_args, apply_setting, apply_prompt, apply_order, apply_sampler, apply_hr_sampler_name, confirm_samplers, apply_checkpoint, apply_refiner, apply_unet, apply_dict, apply_clip_skip, apply_vae, list_lora, apply_lora, apply_lora_strength, apply_te, apply_styles, apply_upscaler, apply_context, apply_detailer, apply_override, apply_processing, apply_options, apply_seed, format_value_add_label, format_value, format_value_join_list, do_nothing, format_nothing # pylint: disable=no-name-in-module, unused-import from modules import shared, errors, scripts, images, processing from modules.ui_components import ToolButton import modules.ui_symbols as symbols From 9126483178e48bc2f3c74d9e03cfc666ec5d5e10 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 2 Dec 2024 11:21:39 -0500 Subject: [PATCH 044/249] add bdia sampler Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 6 +- modules/lora/networks.py | 12 +- modules/schedulers/scheduler_bdia.py | 551 +++++++++++++++++++++++++++ modules/sd_samplers_diffusers.py | 3 + modules/shared.py | 2 +- 5 files changed, 563 insertions(+), 11 deletions(-) create mode 100644 modules/schedulers/scheduler_bdia.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a9109341d..57b08fbe7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,9 +32,12 @@ ### UI and workflow improvements -- **LoRA** handler rewrite +- **LoRA** handler rewrite: - LoRA weights are no longer calculated on-the-fly during model execution, but are pre-calculated at the start this results in perceived overhead on generate startup, but results in overall faster execution as LoRA does not need to be processed on each step + - *note*: LoRA weights backups are required so LoRA can be unapplied, but can take quite a lot of system memory + if you know you will not need to unapply LoRA, you can disable backups in *settings -> networks -> lora fuse* + in which case, you need to reload model to unapply LoRA - **Model loader** improvements: - detect model components on model load fail - allow passing absolute path to model loader @@ -60,6 +63,7 @@ - **Sampler** improvements - Euler FlowMatch: add sigma methods (*karras/exponential/betas*) - DPM FlowMatch: update all and add sigma methods + - BDIA-DDIM: *experimental* ### Fixes diff --git a/modules/lora/networks.py b/modules/lora/networks.py index 48073774c..14fce760a 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -403,7 +403,7 @@ def network_calc_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn. return batch_updown, batch_ex_bias -def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv], updown, ex_bias, apply: bool = True): +def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv], updown, ex_bias): t0 = time.time() weights_backup = getattr(self, "network_weights_backup", None) bias_backup = getattr(self, "network_bias_backup", None) @@ -417,10 +417,7 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn if updown is not None and len(weights_backup.shape) == 4 and weights_backup.shape[1] == 9: # inpainting model. zero pad updown to make channel[1] 4 to 9 updown = torch.nn.functional.pad(updown, (0, 0, 0, 0, 0, 5)) # pylint: disable=not-callable if updown is not None: - if apply: - new_weight = weights_backup.to(devices.device, non_blocking=True) + updown.to(devices.device, non_blocking=True) - else: - new_weight = weights_backup.to(devices.device, non_blocking=True) - updown.to(devices.device, non_blocking=True) + new_weight = weights_backup.to(devices.device, non_blocking=True) + updown.to(devices.device, non_blocking=True) if getattr(self, "quant_type", None) in ['nf4', 'fp4'] and bnb is not None: self.weight = bnb.nn.Params4bit(new_weight, quant_state=self.quant_state, quant_type=self.quant_type, blocksize=self.blocksize) else: @@ -436,10 +433,7 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn else: self.bias = None if ex_bias is not None: - if apply: - new_weight = bias_backup.to(devices.device, non_blocking=True) + ex_bias.to(devices.device, non_blocking=True) - else: - new_weight = bias_backup.to(devices.device, non_blocking=True) - ex_bias.to(devices.device, non_blocking=True) + new_weight = bias_backup.to(devices.device, non_blocking=True) + ex_bias.to(devices.device, non_blocking=True) self.bias = torch.nn.Parameter(new_weight, requires_grad=False) del new_weight else: diff --git a/modules/schedulers/scheduler_bdia.py b/modules/schedulers/scheduler_bdia.py new file mode 100644 index 000000000..bb3e7f9b2 --- /dev/null +++ b/modules/schedulers/scheduler_bdia.py @@ -0,0 +1,551 @@ +# Copyright 2024 Stanford University Team and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DISCLAIMER: This code is strongly influenced by https://github.com/pesser/pytorch_diffusion +# and https://github.com/hojonathanho/diffusion + +import math +from dataclasses import dataclass +from typing import List, Optional, Tuple, Union + +import numpy as np +import torch + +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.utils import BaseOutput +from diffusers.utils.torch_utils import randn_tensor +from diffusers.schedulers.scheduling_utils import KarrasDiffusionSchedulers, SchedulerMixin + + +@dataclass +# Copied from diffusers.schedulers.scheduling_ddpm.DDPMSchedulerOutput with DDPM->DDIM +class DDIMSchedulerOutput(BaseOutput): + """ + Output class for the scheduler's `step` function output. + + Args: + prev_sample (`torch.Tensor` of shape `(batch_size, num_channels, height, width)` for images): + Computed sample `(x_{t-1})` of previous timestep. `prev_sample` should be used as next model input in the + denoising loop. + pred_original_sample (`torch.Tensor` of shape `(batch_size, num_channels, height, width)` for images): + The predicted denoised sample `(x_{0})` based on the model output from the current timestep. + `pred_original_sample` can be used to preview progress or for guidance. + """ + + prev_sample: torch.Tensor + pred_original_sample: Optional[torch.Tensor] = None + + +# Copied from diffusers.schedulers.scheduling_ddpm.betas_for_alpha_bar +def betas_for_alpha_bar( + num_diffusion_timesteps, + max_beta=0.999, + alpha_transform_type="cosine", +): + """ + Create a beta schedule that discretizes the given alpha_t_bar function, which defines the cumulative product of + (1-beta) over time from t = [0,1]. + + Contains a function alpha_bar that takes an argument t and transforms it to the cumulative product of (1-beta) up + to that part of the diffusion process. + + + Args: + num_diffusion_timesteps (`int`): the number of betas to produce. + max_beta (`float`): the maximum beta to use; use values lower than 1 to + prevent singularities. + alpha_transform_type (`str`, *optional*, default to `cosine`): the type of noise schedule for alpha_bar. + Choose from `cosine` or `exp` + + Returns: + betas (`np.ndarray`): the betas used by the scheduler to step the model outputs + """ + if alpha_transform_type == "cosine": + + def alpha_bar_fn(t): + return math.cos((t + 0.008) / 1.008 * math.pi / 2) ** 2 + + elif alpha_transform_type == "exp": + + def alpha_bar_fn(t): + return math.exp(t * -12.0) + + else: + raise ValueError(f"Unsupported alpha_transform_type: {alpha_transform_type}") + + betas = [] + for i in range(num_diffusion_timesteps): + t1 = i / num_diffusion_timesteps + t2 = (i + 1) / num_diffusion_timesteps + betas.append(min(1 - alpha_bar_fn(t2) / alpha_bar_fn(t1), max_beta)) + return torch.tensor(betas, dtype=torch.float32) + + +def rescale_zero_terminal_snr(betas): + """ + Rescales betas to have zero terminal SNR Based on https://arxiv.org/pdf/2305.08891.pdf (Algorithm 1) + + + Args: + betas (`torch.Tensor`): + the betas that the scheduler is being initialized with. + + Returns: + `torch.Tensor`: rescaled betas with zero terminal SNR + """ + # Convert betas to alphas_bar_sqrt + alphas = 1.0 - betas + alphas_cumprod = torch.cumprod(alphas, dim=0) + alphas_bar_sqrt = alphas_cumprod.sqrt() + + # Store old values. + alphas_bar_sqrt_0 = alphas_bar_sqrt[0].clone() + alphas_bar_sqrt_T = alphas_bar_sqrt[-1].clone() + + # Shift so the last timestep is zero. + alphas_bar_sqrt -= alphas_bar_sqrt_T + + # Scale so the first timestep is back to the old value. + alphas_bar_sqrt *= alphas_bar_sqrt_0 / (alphas_bar_sqrt_0 - alphas_bar_sqrt_T) + + # Convert alphas_bar_sqrt to betas + alphas_bar = alphas_bar_sqrt**2 # Revert sqrt + alphas = alphas_bar[1:] / alphas_bar[:-1] # Revert cumprod + alphas = torch.cat([alphas_bar[0:1], alphas]) + betas = 1 - alphas + + return betas + +class BDIA_DDIMScheduler(SchedulerMixin, ConfigMixin): + """ + `DDIMScheduler` extends the denoising procedure introduced in denoising diffusion probabilistic models (DDPMs) with + non-Markovian guidance. + + This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. Check the superclass documentation for the generic + methods the library implements for all schedulers such as loading and saving. + + Args: + num_train_timesteps (`int`, defaults to 1000): + The number of diffusion steps to train the model. + beta_start (`float`, defaults to 0.0001): + The starting `beta` value of inference. + beta_end (`float`, defaults to 0.02): + The final `beta` value. + beta_schedule (`str`, defaults to `"linear"`): + The beta schedule, a mapping from a beta range to a sequence of betas for stepping the model. Choose from + `linear`, `scaled_linear`, or `squaredcos_cap_v2`. + trained_betas (`np.ndarray`, *optional*): + Pass an array of betas directly to the constructor to bypass `beta_start` and `beta_end`. + clip_sample (`bool`, defaults to `True`): + Clip the predicted sample for numerical stability. + clip_sample_range (`float`, defaults to 1.0): + The maximum magnitude for sample clipping. Valid only when `clip_sample=True`. + set_alpha_to_one (`bool`, defaults to `True`): + Each diffusion step uses the alphas product value at that step and at the previous one. For the final step + there is no previous alpha. When this option is `True` the previous alpha product is fixed to `1`, + otherwise it uses the alpha value at step 0. + steps_offset (`int`, defaults to 0): + An offset added to the inference steps, as required by some model families. + prediction_type (`str`, defaults to `epsilon`, *optional*): + Prediction type of the scheduler function; can be `epsilon` (predicts the noise of the diffusion process), + `sample` (directly predicts the noisy sample`) or `v_prediction` (see section 2.4 of [Imagen + Video](https://imagen.research.google/video/paper.pdf) paper). + thresholding (`bool`, defaults to `False`): + Whether to use the "dynamic thresholding" method. This is unsuitable for latent-space diffusion models such + as Stable Diffusion. + dynamic_thresholding_ratio (`float`, defaults to 0.995): + The ratio for the dynamic thresholding method. Valid only when `thresholding=True`. + sample_max_value (`float`, defaults to 1.0): + The threshold value for dynamic thresholding. Valid only when `thresholding=True`. + timestep_spacing (`str`, defaults to `"leading"`): + The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information. + rescale_betas_zero_snr (`bool`, defaults to `False`): + Whether to rescale the betas to have zero terminal SNR. This enables the model to generate very bright and + dark samples instead of limiting it to samples with medium brightness. Loosely related to + [`--offset_noise`](https://github.com/huggingface/diffusers/blob/74fd735eb073eb1d774b1ab4154a0876eb82f055/examples/dreambooth/train_dreambooth.py#L506). + """ + + _compatibles = [e.name for e in KarrasDiffusionSchedulers] + order = 1 + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + beta_start: float = 0.0001, + beta_end: float = 0.02, + beta_schedule: str = "linear", + trained_betas: Optional[Union[np.ndarray, List[float]]] = None, + clip_sample: bool = True, + set_alpha_to_one: bool = True, #was True + steps_offset: int = 0, + prediction_type: str = "epsilon", + thresholding: bool = False, + dynamic_thresholding_ratio: float = 0.995, + clip_sample_range: float = 1.0, + sample_max_value: float = 1.0, + timestep_spacing: str = "leading", #leading + rescale_betas_zero_snr: bool = False, + gamma: float = 1.0, + + ): + if trained_betas is not None: + self.betas = torch.tensor(trained_betas, dtype=torch.float32) + elif beta_schedule == "linear": + self.betas = torch.linspace(beta_start, beta_end, num_train_timesteps, dtype=torch.float32) + elif beta_schedule == "scaled_linear": + # this schedule is very specific to the latent diffusion model. + self.betas = torch.linspace(beta_start**0.5, beta_end**0.5, num_train_timesteps, dtype=torch.float32) ** 2 + elif beta_schedule == "squaredcos_cap_v2": + # Glide cosine schedule + self.betas = betas_for_alpha_bar(num_train_timesteps) + else: + raise NotImplementedError(f"{beta_schedule} is not implemented for {self.__class__}") + + # Rescale for zero SNR + if rescale_betas_zero_snr: + self.betas = rescale_zero_terminal_snr(self.betas) + + self.alphas = 1.0 - self.betas #may have to add something for last step + + self.alphas_cumprod = torch.cumprod(self.alphas, dim=0) + # At every step in ddim, we are looking into the previous alphas_cumprod + # For the final step, there is no previous alphas_cumprod because we are already at 0 + # `set_alpha_to_one` decides whether we set this parameter simply to one or + # whether we use the final alpha of the "non-previous" one. + self.final_alpha_cumprod = torch.tensor(1.0) if set_alpha_to_one else self.alphas_cumprod[0] + + # standard deviation of the initial noise distribution + self.init_noise_sigma = 1.0 + + # setable values + self.num_inference_steps = None + self.timesteps = torch.from_numpy(np.arange(0, num_train_timesteps)[::-1].copy().astype(np.int64)) + self.next_sample = [] + self.BDIA = False + + + def scale_model_input(self, sample: torch.Tensor, timestep: Optional[int] = None) -> torch.Tensor: + """ + Ensures interchangeability with schedulers that need to scale the denoising model input depending on the + current timestep. + + Args: + sample (`torch.Tensor`): + The input sample. + timestep (`int`, *optional*): + The current timestep in the diffusion chain. + + Returns: + `torch.Tensor`: + A scaled input sample. + """ + return sample + + def _get_variance(self, timestep, prev_timestep): + alpha_prod_t = self.alphas_cumprod[timestep] + alpha_prod_t_prev = self.alphas_cumprod[prev_timestep] if prev_timestep >= 0 else self.final_alpha_cumprod + beta_prod_t = 1 - alpha_prod_t + beta_prod_t_prev = 1 - alpha_prod_t_prev + + variance = (beta_prod_t_prev / beta_prod_t) * (1 - alpha_prod_t / alpha_prod_t_prev) + + return variance + + # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler._threshold_sample + def _threshold_sample(self, sample: torch.Tensor) -> torch.Tensor: + """ + "Dynamic thresholding: At each sampling step we set s to a certain percentile absolute pixel value in xt0 (the + prediction of x_0 at timestep t), and if s > 1, then we threshold xt0 to the range [-s, s] and then divide by + s. Dynamic thresholding pushes saturated pixels (those near -1 and 1) inwards, thereby actively preventing + pixels from saturation at each step. We find that dynamic thresholding results in significantly better + photorealism as well as better image-text alignment, especially when using very large guidance weights." + + https://arxiv.org/abs/2205.11487 + """ + dtype = sample.dtype + batch_size, channels, *remaining_dims = sample.shape + + if dtype not in (torch.float32, torch.float64): + sample = sample.float() # upcast for quantile calculation, and clamp not implemented for cpu half + + # Flatten sample for doing quantile calculation along each image + sample = sample.reshape(batch_size, channels * np.prod(remaining_dims)) + + abs_sample = sample.abs() # "a certain percentile absolute pixel value" + + s = torch.quantile(abs_sample, self.config.dynamic_thresholding_ratio, dim=1) + s = torch.clamp( + s, min=1, max=self.config.sample_max_value + ) # When clamped to min=1, equivalent to standard clipping to [-1, 1] + s = s.unsqueeze(1) # (batch_size, 1) because clamp will broadcast along dim=0 + sample = torch.clamp(sample, -s, s) / s # "we threshold xt0 to the range [-s, s] and then divide by s" + + sample = sample.reshape(batch_size, channels, *remaining_dims) + sample = sample.to(dtype) + + return sample + + def set_timesteps(self, num_inference_steps: int, device: Union[str, torch.device] = None): + """ + Sets the discrete timesteps used for the diffusion chain (to be run before inference). + + Args: + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. + """ + + if num_inference_steps > self.config.num_train_timesteps: + raise ValueError( + f"`num_inference_steps`: {num_inference_steps} cannot be larger than `self.config.train_timesteps`:" + f" {self.config.num_train_timesteps} as the unet model trained with this scheduler can only handle" + f" maximal {self.config.num_train_timesteps} timesteps." + ) + + self.num_inference_steps = num_inference_steps + + # "linspace", "leading", "trailing" corresponds to annotation of Table 2. of https://arxiv.org/abs/2305.08891 + if self.config.timestep_spacing == "linspace": + timesteps = ( + np.linspace(0, self.config.num_train_timesteps - 1, num_inference_steps) + .round()[::-1] + .copy() + .astype(np.int64) + ) + elif self.config.timestep_spacing == "leading": + step_ratio = self.config.num_train_timesteps // self.num_inference_steps + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + timesteps = (np.arange(0, num_inference_steps) * step_ratio).round()[::-1].copy().astype(np.int64) + timesteps += self.config.steps_offset + elif self.config.timestep_spacing == "trailing": + step_ratio = self.config.num_train_timesteps / self.num_inference_steps + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + timesteps = np.round(np.arange(self.config.num_train_timesteps, 0, -step_ratio)).astype(np.int64) + timesteps -= 1 + else: + raise ValueError( + f"{self.config.timestep_spacing} is not supported. Please make sure to choose one of 'leading' or 'trailing'." + ) + + self.timesteps = torch.from_numpy(timesteps).to(device) + + def step( + self, + model_output: torch.Tensor, + timestep: int, + sample: torch.Tensor, + eta: float = 0.0, + use_clipped_model_output: bool = False, + generator=None, + variance_noise: Optional[torch.Tensor] = None, + return_dict: bool = True, + debug: bool = False, + ) -> Union[DDIMSchedulerOutput, Tuple]: + """ + Predict the sample from the previous timestep by reversing the SDE. + + Args: + model_output (torch.Tensor): Direct output from learned diffusion model + timestep (int): Current discrete timestep in the diffusion chain + sample (torch.Tensor): Current instance of sample created by diffusion process + eta (float): Weight of noise for added noise in diffusion step + use_clipped_model_output (bool): Whether to use clipped model output + generator (torch.Generator, optional): Random number generator + variance_noise (torch.Tensor, optional): Pre-generated noise for variance + return_dict (bool): Whether to return as DDIMSchedulerOutput or tuple + debug (bool): Whether to print debug information + """ + if self.num_inference_steps is None: + raise ValueError("Number of inference steps is 'None', run 'set_timesteps' first") + + # Calculate timesteps + step_size = self.config.num_train_timesteps // self.num_inference_steps + prev_timestep = timestep - step_size + next_timestep = timestep + step_size + + if debug: + print("\n=== Timestep Information ===") + print(f"Current timestep: {timestep}") + print(f"Previous timestep: {prev_timestep}") + print(f"Next timestep: {next_timestep}") + print(f"Step size: {step_size}") + + # Pre-compute alpha and variance values + alpha_prod_t = self.alphas_cumprod[timestep] + alpha_prod_t_prev = self.alphas_cumprod[prev_timestep] if prev_timestep >= 0 else self.final_alpha_cumprod + variance = self._get_variance(timestep, prev_timestep) + std_dev_t = eta * variance ** 0.5 + + # Compute required values + alpha_i = alpha_prod_t ** 0.5 + alpha_i_minus_1 = alpha_prod_t_prev ** 0.5 + sigma_i = (1 - alpha_prod_t) ** 0.5 + sigma_i_minus_1 = (1 - alpha_prod_t_prev - std_dev_t**2) ** 0.5 + + if debug: + print("\n=== Alpha Values ===") + print(f"alpha_i: {alpha_i}") + print(f"alpha_i_minus_1: {alpha_i_minus_1}") + print(f"sigma_i: {sigma_i}") + print(f"sigma_i_minus_1: {sigma_i_minus_1}") + + # Predict original sample based on prediction type + if self.config.prediction_type == "epsilon": + pred_original_sample = (sample - sigma_i * model_output) / alpha_i + pred_epsilon = model_output + if debug: + print("\nPrediction type: epsilon") + elif self.config.prediction_type == "sample": + pred_original_sample = model_output + pred_epsilon = (sample - alpha_i * pred_original_sample) / sigma_i + if debug: + print("\nPrediction type: sample") + elif self.config.prediction_type == "v_prediction": + pred_original_sample = alpha_i * sample - sigma_i * model_output + pred_epsilon = alpha_i * model_output + sigma_i * sample + if debug: + print("\nPrediction type: v_prediction") + else: + raise ValueError( + f"prediction_type {self.config.prediction_type} must be one of `epsilon`, `sample`, or `v_prediction`" + ) + + # Apply thresholding or clipping if configured + if self.config.thresholding: + if debug: + print("\nApplying thresholding") + pred_original_sample = self._threshold_sample(pred_original_sample) + elif self.config.clip_sample: + if debug: + print("\nApplying clipping") + pred_original_sample = pred_original_sample.clamp( + -self.config.clip_sample_range, self.config.clip_sample_range + ) + + # Recompute pred_epsilon if using clipped model output + if use_clipped_model_output: + if debug: + print("\nUsing clipped model output") + pred_epsilon = (sample - alpha_i * pred_original_sample) / sigma_i + + # Compute DDIM step + ddim_step = alpha_i_minus_1 * pred_original_sample + sigma_i_minus_1 * pred_epsilon + + # Handle initial DDIM step or BDIA steps + if len(self.next_sample) == 0: + if debug: + print("\nFirst iteration (DDIM)") + self.update_next_sample_BDIA(sample) + self.update_next_sample_BDIA(ddim_step) + else: + if debug: + print("\nBDIA step") + # BDIA implementation + alpha_prod_t_next = self.alphas_cumprod[next_timestep] + alpha_i_plus_1 = alpha_prod_t_next ** 0.5 + sigma_i_plus_1 = (1 - alpha_prod_t_next) ** 0.5 + + if debug: + print(f"alpha_i_plus_1: {alpha_i_plus_1}") + print(f"sigma_i_plus_1: {sigma_i_plus_1}") + + a = alpha_i_plus_1 * pred_original_sample + sigma_i_plus_1 * pred_epsilon + bdia_step = ( + self.config.gamma * self.next_sample[-2] + + ddim_step - + (self.config.gamma * a) + ) + self.update_next_sample_BDIA(bdia_step) + + prev_sample = self.next_sample[-1] + + # Apply variance noise if eta > 0 + if eta > 0: + if debug: + print(f"\nApplying variance noise with eta: {eta}") + + if variance_noise is not None and generator is not None: + raise ValueError( + "Cannot pass both generator and variance_noise. Use either `generator` or `variance_noise`." + ) + + if variance_noise is None: + variance_noise = randn_tensor( + model_output.shape, + generator=generator, + device=model_output.device, + dtype=model_output.dtype + ) + prev_sample = prev_sample + std_dev_t * variance_noise + + if not return_dict: + return (prev_sample,) + + return DDIMSchedulerOutput(prev_sample=prev_sample, pred_original_sample=pred_original_sample) + + def add_noise( + self, + original_samples: torch.Tensor, + noise: torch.Tensor, + timesteps: torch.IntTensor, + ) -> torch.Tensor: + # Make sure alphas_cumprod and timestep have same device and dtype as original_samples + # Move the self.alphas_cumprod to device to avoid redundant CPU to GPU data movement + # for the subsequent add_noise calls + self.alphas_cumprod = self.alphas_cumprod.to(device=original_samples.device) + alphas_cumprod = self.alphas_cumprod.to(dtype=original_samples.dtype) + timesteps = timesteps.to(original_samples.device) + + sqrt_alpha_prod = alphas_cumprod[timesteps] ** 0.5 + sqrt_alpha_prod = sqrt_alpha_prod.flatten() + while len(sqrt_alpha_prod.shape) < len(original_samples.shape): + sqrt_alpha_prod = sqrt_alpha_prod.unsqueeze(-1) + + sqrt_one_minus_alpha_prod = (1 - alphas_cumprod[timesteps]) ** 0.5 + sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.flatten() + while len(sqrt_one_minus_alpha_prod.shape) < len(original_samples.shape): + sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.unsqueeze(-1) + + noisy_samples = sqrt_alpha_prod * original_samples + sqrt_one_minus_alpha_prod * noise + return noisy_samples + + # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler.get_velocity + def get_velocity(self, sample: torch.Tensor, noise: torch.Tensor, timesteps: torch.IntTensor) -> torch.Tensor: + # Make sure alphas_cumprod and timestep have same device and dtype as sample + self.alphas_cumprod = self.alphas_cumprod.to(device=sample.device) + alphas_cumprod = self.alphas_cumprod.to(dtype=sample.dtype) + timesteps = timesteps.to(sample.device) + + sqrt_alpha_prod = alphas_cumprod[timesteps] ** 0.5 + sqrt_alpha_prod = sqrt_alpha_prod.flatten() + while len(sqrt_alpha_prod.shape) < len(sample.shape): + sqrt_alpha_prod = sqrt_alpha_prod.unsqueeze(-1) + + sqrt_one_minus_alpha_prod = (1 - alphas_cumprod[timesteps]) ** 0.5 + sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.flatten() + while len(sqrt_one_minus_alpha_prod.shape) < len(sample.shape): + sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.unsqueeze(-1) + + velocity = sqrt_alpha_prod * noise - sqrt_one_minus_alpha_prod * sample + return velocity + + def update_next_sample_BDIA(self, new_value): + self.next_sample.append(new_value.clone()) + + + def __len__(self): + return self.config.num_train_timesteps \ No newline at end of file diff --git a/modules/sd_samplers_diffusers.py b/modules/sd_samplers_diffusers.py index 4672df92e..7c23d4342 100644 --- a/modules/sd_samplers_diffusers.py +++ b/modules/sd_samplers_diffusers.py @@ -52,6 +52,7 @@ from modules.schedulers.scheduler_dc import DCSolverMultistepScheduler # pylint: disable=ungrouped-imports from modules.schedulers.scheduler_vdm import VDMScheduler # pylint: disable=ungrouped-imports from modules.schedulers.scheduler_dpm_flowmatch import FlowMatchDPMSolverMultistepScheduler # pylint: disable=ungrouped-imports + from modules.schedulers.scheduler_bdia import BDIA_DDIMScheduler # pylint: disable=ungrouped-imports except Exception as e: shared.log.error(f'Diffusers import error: version={diffusers.__version__} error: {e}') if os.environ.get('SD_SAMPLER_DEBUG', None) is not None: @@ -97,6 +98,7 @@ 'VDM Solver': { 'clip_sample_range': 2.0, }, 'LCM': { 'beta_start': 0.00085, 'beta_end': 0.012, 'beta_schedule': "scaled_linear", 'set_alpha_to_one': True, 'rescale_betas_zero_snr': False, 'thresholding': False, 'timestep_spacing': 'linspace' }, 'TCD': { 'set_alpha_to_one': True, 'rescale_betas_zero_snr': False, 'beta_schedule': 'scaled_linear' }, + 'BDIA DDIM': { 'clip_sample': False, 'set_alpha_to_one': True, 'steps_offset': 0, 'clip_sample_range': 1.0, 'sample_max_value': 1.0, 'timestep_spacing': 'leading', 'rescale_betas_zero_snr': False, 'thresholding': False, 'gamma': 1.0 }, 'PNDM': { 'skip_prk_steps': False, 'set_alpha_to_one': False, 'steps_offset': 0, 'timestep_spacing': 'linspace' }, 'IPNDM': { }, @@ -142,6 +144,7 @@ sd_samplers_common.SamplerData('SA Solver', lambda model: DiffusionSampler('SA Solver', SASolverScheduler, model), [], {}), sd_samplers_common.SamplerData('DC Solver', lambda model: DiffusionSampler('DC Solver', DCSolverMultistepScheduler, model), [], {}), sd_samplers_common.SamplerData('VDM Solver', lambda model: DiffusionSampler('VDM Solver', VDMScheduler, model), [], {}), + sd_samplers_common.SamplerData('BDIA DDIM', lambda model: DiffusionSampler('BDIA DDIM g=0', BDIA_DDIMScheduler, model), [], {}), sd_samplers_common.SamplerData('PNDM', lambda model: DiffusionSampler('PNDM', PNDMScheduler, model), [], {}), sd_samplers_common.SamplerData('IPNDM', lambda model: DiffusionSampler('IPNDM', IPNDMScheduler, model), [], {}), diff --git a/modules/shared.py b/modules/shared.py index 8f809fa8b..f5c311f21 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -20,7 +20,7 @@ from modules.paths import models_path, script_path, data_path, sd_configs_path, sd_default_config, sd_model_file, default_sd_model_file, extensions_dir, extensions_builtin_dir # pylint: disable=W0611 from modules.dml import memory_providers, default_memory_provider, directml_do_hijack from modules.onnx_impl import initialize_onnx, execution_providers -from modules.memstats import memory_stats +from modules.memstats import memory_stats, ram_stats # pylint: disable=unused-import from modules.ui_components import DropdownEditable import modules.interrogate import modules.memmon From 9f8622491eccd352b937013235bfeaa4abce073a Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sat, 30 Nov 2024 00:55:22 +0300 Subject: [PATCH 045/249] Update OpenVINO to 2024.5.0 --- CHANGELOG.md | 1 + installer.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 57b08fbe7..8091176e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ style-aligned applies selected attention layers uniformly to all images to achive consistency can be used with or without input image in which case first prompt is used to establish baseline *note:* all prompts are processes as a single batch, so vram is limiting factor +- **OpenVINO**: update to 2024.5.0 ### UI and workflow improvements diff --git a/installer.py b/installer.py index 8fb6d9683..c849ac5b6 100644 --- a/installer.py +++ b/installer.py @@ -640,7 +640,7 @@ def install_ipex(torch_command): # os.environ.setdefault('TENSORFLOW_PACKAGE', 'tensorflow==2.15.1 intel-extension-for-tensorflow[xpu]==2.15.0.1') else: torch_command = os.environ.get('TORCH_COMMAND', '--pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/xpu') # torchvision doesn't exist on test/stable branch for windows - install(os.environ.get('OPENVINO_PACKAGE', 'openvino==2024.3.0'), 'openvino', ignore=True) + install(os.environ.get('OPENVINO_PACKAGE', 'openvino==2024.5.0'), 'openvino', ignore=True) install('nncf==2.7.0', 'nncf', ignore=True) install(os.environ.get('ONNXRUNTIME_PACKAGE', 'onnxruntime-openvino'), 'onnxruntime-openvino', ignore=True) return torch_command @@ -650,7 +650,7 @@ def install_openvino(torch_command): check_python(supported_minors=[8, 9, 10, 11, 12], reason='OpenVINO backend requires Python 3.9, 3.10 or 3.11') log.info('OpenVINO: selected') torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.3.1+cpu torchvision==0.18.1+cpu --index-url https://download.pytorch.org/whl/cpu') - install(os.environ.get('OPENVINO_PACKAGE', 'openvino==2024.3.0'), 'openvino') + install(os.environ.get('OPENVINO_PACKAGE', 'openvino==2024.5.0'), 'openvino') install(os.environ.get('ONNXRUNTIME_PACKAGE', 'onnxruntime-openvino'), 'onnxruntime-openvino', ignore=True) install('nncf==2.12.0', 'nncf') os.environ.setdefault('PYTORCH_TRACING_MODE', 'TORCHFX') From 3df4e8789ba268a9b899530cfbfb0c3a9b580d83 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sat, 30 Nov 2024 01:15:49 +0300 Subject: [PATCH 046/249] ZLUDA enable Dynamic attention by default --- modules/shared.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/shared.py b/modules/shared.py index f5c311f21..6b5c3f9f1 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -450,16 +450,16 @@ def get_default_modes(): default_cross_attention = "Scaled-Dot-Product" if native else "Doggettx's" elif devices.backend == "mps": default_cross_attention = "Scaled-Dot-Product" if native else "Doggettx's" - else: # cuda, rocm, ipex, openvino - default_cross_attention ="Scaled-Dot-Product" + else: # cuda, rocm, zluda, ipex, openvino + default_cross_attention = "Scaled-Dot-Product" if devices.backend == "rocm": default_sdp_options = ['Memory attention', 'Math attention'] elif devices.backend == "zluda": - default_sdp_options = ['Math attention'] + default_sdp_options = ['Math attention', 'Dynamic attention'] else: default_sdp_options = ['Flash attention', 'Memory attention', 'Math attention'] - if (cmd_opts.lowvram or cmd_opts.medvram) and ('Flash attention' not in default_sdp_options): + if (cmd_opts.lowvram or cmd_opts.medvram) and ('Flash attention' not in default_sdp_options and 'Dynamic attention' not in default_sdp_options): default_sdp_options.append('Dynamic attention') return default_offload_mode, default_cross_attention, default_sdp_options From d0a81c161fd10510ce5bf0c7522c34bd192c768b Mon Sep 17 00:00:00 2001 From: Pablo Hellmann Date: Sun, 1 Dec 2024 01:12:28 +0100 Subject: [PATCH 047/249] prompt token counter fix --- javascript/black-teal-reimagined.css | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/javascript/black-teal-reimagined.css b/javascript/black-teal-reimagined.css index b7567ce75..0a985b786 100644 --- a/javascript/black-teal-reimagined.css +++ b/javascript/black-teal-reimagined.css @@ -985,9 +985,10 @@ svg.feather.feather-image, #txt2img_token_counter, #txt2img_negative_token_counter { display: flex; - flex-direction: column; - justify-content: space-evenly; - padding: 5px; + flex-direction: row; + padding-top: 1px; + opacity: 0.6; + z-index: 99; } #txt2img_prompt_container { From dc1c9074c00c38ab792a771a9c08deb6bcce2311 Mon Sep 17 00:00:00 2001 From: Pablo Hellmann Date: Sun, 1 Dec 2024 01:23:13 +0100 Subject: [PATCH 048/249] small changes and removed useless css --- javascript/black-teal-reimagined.css | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/javascript/black-teal-reimagined.css b/javascript/black-teal-reimagined.css index 0a985b786..25f4a3f2c 100644 --- a/javascript/black-teal-reimagined.css +++ b/javascript/black-teal-reimagined.css @@ -392,13 +392,13 @@ input[type='range']::-moz-range-track { } /* Form Styles */ -div.form { +div.form, #txt2img_seed_row, #txt2img_subseed_row { border-width: 0; box-shadow: var(--shadow-md); background: var(--background-fill-primary); border-bottom: 3px solid var(--highlight-color); padding: 3px; - border-radius: var(--radius-md); + border-radius: var(--radius-lg); margin: 1px; } @@ -700,17 +700,6 @@ svg.feather.feather-image, height: 2.4em; } -#footer, -#style_pos_col, -#style_neg_col, -#roll_col, -#extras_upscaler_2, -#extras_upscaler_2_visibility, -#txt2img_seed_resize_from_w, -#txt2img_seed_resize_from_h { - display: none; -} - #save-animation { border-radius: var(--radius-sm) !important; margin-bottom: 16px; From 90e5dcc730b3820cfb225495de676de07379eff2 Mon Sep 17 00:00:00 2001 From: Pablo Hellmann Date: Sun, 1 Dec 2024 02:19:47 +0100 Subject: [PATCH 049/249] darker colors and fancy live preview --- javascript/black-teal-reimagined.css | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/javascript/black-teal-reimagined.css b/javascript/black-teal-reimagined.css index 25f4a3f2c..7b3a281ee 100644 --- a/javascript/black-teal-reimagined.css +++ b/javascript/black-teal-reimagined.css @@ -54,8 +54,8 @@ html { /* Background Colors */ --background-color: var(--neutral-950); - --background-fill-primary: var(--neutral-700); - --input-background-fill: var(--neutral-800); + --background-fill-primary: var(--neutral-800); + --input-background-fill: var(--neutral-900); /* Padding and Borders */ --input-padding: 4px; @@ -402,6 +402,30 @@ div.form, #txt2img_seed_row, #txt2img_subseed_row { margin: 1px; } +/* Image preview styling*/ +#txt2img_gallery { + background: var(--background-fill-primary); + padding: 5px; + margin: 0px; +} + +@keyframes colorChange { + 0% { + background-color: var(--neutral-800); + } + 50% { + background-color: var(--neutral-700); + } + 100% { + background-color: var(--neutral-800); + } +} + +.livePreview { + animation: colorChange 3s ease-in-out infinite; /* Adjust the duration as needed */ + padding: 5px; +} + /* Gradio Style Classes */ fieldset .gr-block.gr-box, label.block span { From 990d64347615c67e56a5cad51e800ae9e773f9af Mon Sep 17 00:00:00 2001 From: Pablo Hellmann Date: Sun, 1 Dec 2024 02:39:56 +0100 Subject: [PATCH 050/249] Fancy loader --- javascript/black-teal-reimagined.css | 111 ++++++++++++++++++++++++++- 1 file changed, 109 insertions(+), 2 deletions(-) diff --git a/javascript/black-teal-reimagined.css b/javascript/black-teal-reimagined.css index 7b3a281ee..28176d247 100644 --- a/javascript/black-teal-reimagined.css +++ b/javascript/black-teal-reimagined.css @@ -994,8 +994,116 @@ svg.feather.feather-image, height: 100%; } -/* Token counters styling */ +/* loader */ +.splash { + position: fixed; + top: 0; + left: 0; + width: 100vw; + height: 100vh; + z-index: 1000; + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + background-color: rgba(0, 0, 0, 0.8); +} + +.motd { + margin-top: 1em; + color: var(--body-text-color-subdued); + font-family: monospace; + font-variant: all-petite-caps; + font-size: 1.2em; +} + +.splash-img { + margin: 0; + width: 512px; + height: 512px; + background-repeat: no-repeat; + animation: color 8s infinite alternate, move 3s infinite alternate; +} + +.loading { + color: white; + position: border-box; + top: 85%; + font-size: 1.5em; +} + +.loader { + width: 100px; + height: 100px; + border: var(--spacing-md) solid transparent; + border-radius: 50%; + border-top: var(--spacing-md) solid var(--primary-600); + animation: spin 2s linear infinite, pulse 1.5s ease-in-out infinite; + position: border-box; +} + +.loader::before, +.loader::after { + content: ""; + position: absolute; + top: 6px; + bottom: 6px; + left: 6px; + right: 6px; + border-radius: 50%; + border: var(--spacing-md) solid transparent; +} + +.loader::before { + border-top-color: var(--primary-900); + animation: spin 3s linear infinite; +} + +.loader::after { + border-top-color: var(--primary-300); + animation: spin 1.5s linear infinite; +} +@keyframes move { + 0% { + transform: translateY(0); + } + 50% { + transform: translateY(-10px); + } + 100% { + transform: translateY(0); + } +} + +@keyframes spin { + from { + transform: rotate(0deg); + } + to { + transform: rotate(360deg); + } +} + +@keyframes pulse { + 0%, 100% { + transform: scale(1); + } + 50% { + transform: scale(1.1); + } +} + +@keyframes color { + 0% { + filter: hue-rotate(0deg); + } + 100% { + filter: hue-rotate(360deg); + } +} + +/* Token counters styling */ #txt2img_token_counter, #txt2img_negative_token_counter { display: flex; flex-direction: row; @@ -1063,7 +1171,6 @@ svg.feather.feather-image, --input-radius: var(--radius-lg); --input-text-size: var(--text-md); --input-text-weight: 400; - --loader-color: var(--color-accent); --prose-text-size: var(--text-md); --prose-text-weight: 400; --prose-header-text-weight: 400; From 798622efe2dcb399ce52865ed86a12f56d58e289 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 2 Dec 2024 11:30:30 -0500 Subject: [PATCH 051/249] update light theme Signed-off-by: Vladimir Mandic --- extensions-builtin/sdnext-modernui | 2 +- javascript/light-teal.css | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/extensions-builtin/sdnext-modernui b/extensions-builtin/sdnext-modernui index f083ce41a..3008cee4b 160000 --- a/extensions-builtin/sdnext-modernui +++ b/extensions-builtin/sdnext-modernui @@ -1 +1 @@ -Subproject commit f083ce41a9f18b500f26745ea9e86855e509d2cb +Subproject commit 3008cee4b67bb00f8f1a4fe4510ec27ba92aa418 diff --git a/javascript/light-teal.css b/javascript/light-teal.css index 28bf03e6f..174622e52 100644 --- a/javascript/light-teal.css +++ b/javascript/light-teal.css @@ -20,9 +20,9 @@ --body-text-color: var(--neutral-800); --body-text-color-subdued: var(--neutral-600); --background-color: #FFFFFF; - --background-fill-primary: var(--neutral-400); + --background-fill-primary: var(--neutral-300); --input-padding: 4px; - --input-background-fill: var(--neutral-300); + --input-background-fill: var(--neutral-200); --input-shadow: 2px 2px 2px 2px var(--neutral-500); --button-secondary-text-color: black; --button-secondary-background-fill: linear-gradient(to bottom right, var(--neutral-200), var(--neutral-500)); @@ -291,8 +291,8 @@ svg.feather.feather-image, .feather .feather-image { display: none } --slider-color: ; --stat-background-fill: linear-gradient(to right, var(--primary-400), var(--primary-600)); --table-border-color: var(--neutral-700); - --table-even-background-fill: #222222; - --table-odd-background-fill: #333333; + --table-even-background-fill: #FFFFFF; + --table-odd-background-fill: #CCCCCC; --table-radius: var(--radius-lg); --table-row-focus: var(--color-accent-soft); } From 80fd0873512731908df7e264da78f3661d5b691b Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 2 Dec 2024 12:26:27 -0500 Subject: [PATCH 052/249] update requirements and notes Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 9 +++++++-- TODO.md | 3 ++- requirements.txt | 2 +- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8091176e8..748ac6f04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,13 +29,13 @@ style-aligned applies selected attention layers uniformly to all images to achive consistency can be used with or without input image in which case first prompt is used to establish baseline *note:* all prompts are processes as a single batch, so vram is limiting factor -- **OpenVINO**: update to 2024.5.0 ### UI and workflow improvements - **LoRA** handler rewrite: - LoRA weights are no longer calculated on-the-fly during model execution, but are pre-calculated at the start this results in perceived overhead on generate startup, but results in overall faster execution as LoRA does not need to be processed on each step + thanks @AI-Casanova - *note*: LoRA weights backups are required so LoRA can be unapplied, but can take quite a lot of system memory if you know you will not need to unapply LoRA, you can disable backups in *settings -> networks -> lora fuse* in which case, you need to reload model to unapply LoRA @@ -51,6 +51,7 @@ - faster and more compatible *balanced* mode - balanced offload: units are now in percentage instead of bytes - balanced offload: add both high and low watermark + *note*: balanced offload is recommended method for offload when using any large models such as sd35 or flux - **UI**: - improved stats on generate completion - improved live preview display and performance @@ -60,7 +61,11 @@ - control: optionn to hide input column - control: add stats - browser -> server logging framework - - add addtional themes: `black-reimagined` + - add addtional themes: `black-reimagined`, thanks @Artheriax + +### Updates + +- **OpenVINO**: update to 2024.5.0 - **Sampler** improvements - Euler FlowMatch: add sigma methods (*karras/exponential/betas*) - DPM FlowMatch: update all and add sigma methods diff --git a/TODO.md b/TODO.md index 73008039d..90372e41f 100644 --- a/TODO.md +++ b/TODO.md @@ -9,7 +9,8 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma - Flux IPAdapter: - Flux NF4: - SANA: -- LTX-Video: +- LTX-Video: +- TorchAO: ## Other diff --git a/requirements.txt b/requirements.txt index 68c64954d..52cc7b2d4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -52,7 +52,7 @@ numba==0.59.1 protobuf==4.25.3 pytorch_lightning==1.9.4 tokenizers==0.20.3 -transformers==4.46.2 +transformers==4.46.3 urllib3==1.26.19 Pillow==10.4.0 pillow-jxl-plugin==1.3.0 From 66de64d27106198fc22204fd7be45618ecb99517 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 2 Dec 2024 16:27:22 -0500 Subject: [PATCH 053/249] remove tracemalloc Signed-off-by: Vladimir Mandic --- launch.py | 3 --- modules/cmd_args.py | 1 - modules/processing.py | 8 -------- 3 files changed, 12 deletions(-) diff --git a/launch.py b/launch.py index 5c8a6051a..e00da58c7 100755 --- a/launch.py +++ b/launch.py @@ -192,9 +192,6 @@ def main(): global args # pylint: disable=global-statement installer.ensure_base_requirements() init_args() # setup argparser and default folders - if args.malloc: - import tracemalloc - tracemalloc.start() installer.args = args installer.setup_logging() installer.log.info('Starting SD.Next') diff --git a/modules/cmd_args.py b/modules/cmd_args.py index cb4e5fc16..752ad02c0 100644 --- a/modules/cmd_args.py +++ b/modules/cmd_args.py @@ -26,7 +26,6 @@ def main_args(): group_diag.add_argument("--no-hashing", default=os.environ.get("SD_NOHASHING", False), action='store_true', help="Disable hashing of checkpoints, default: %(default)s") group_diag.add_argument("--no-metadata", default=os.environ.get("SD_NOMETADATA", False), action='store_true', help="Disable reading of metadata from models, default: %(default)s") group_diag.add_argument("--profile", default=os.environ.get("SD_PROFILE", False), action='store_true', help="Run profiler, default: %(default)s") - group_diag.add_argument("--malloc", default=os.environ.get("SD_PROFILE", False), action='store_true', help="Trace memory ops, default: %(default)s") group_diag.add_argument("--disable-queue", default=os.environ.get("SD_DISABLEQUEUE", False), action='store_true', help="Disable queues, default: %(default)s") group_diag.add_argument('--debug', default=os.environ.get("SD_DEBUG", False), action='store_true', help = "Run installer with debug logging, default: %(default)s") diff --git a/modules/processing.py b/modules/processing.py index 095eba54c..57512850a 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -475,13 +475,5 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: if not p.disable_extra_networks: shared.log.info(f'Processed: images={len(output_images)} its={(p.steps * len(output_images)) / (t1 - t0):.2f} time={t1-t0:.2f} timers={timer.process.dct(min_time=0.02)} memory={memstats.memory_stats()}') - if shared.cmd_opts.malloc: - import tracemalloc - snapshot = tracemalloc.take_snapshot() - stats = snapshot.statistics('lineno') - shared.log.debug('Profile malloc:') - for stat in stats[:20]: - frame = stat.traceback[0] - shared.log.debug(f' file="{frame.filename}":{frame.lineno} size={stat.size}') devices.torch_gc(force=True) return processed From 736294e22ebb97365135d7178827bdca79b83750 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 3 Dec 2024 08:52:53 -0500 Subject: [PATCH 054/249] samplers add custom sigma Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 1 + installer.py | 2 +- modules/processing_args.py | 15 +++++++++++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 748ac6f04..51d61aa46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -68,6 +68,7 @@ - **OpenVINO**: update to 2024.5.0 - **Sampler** improvements - Euler FlowMatch: add sigma methods (*karras/exponential/betas*) + - Euler FlowMatch: allow using timestep presets to set sigmas - DPM FlowMatch: update all and add sigma methods - BDIA-DDIM: *experimental* diff --git a/installer.py b/installer.py index c849ac5b6..e03512301 100644 --- a/installer.py +++ b/installer.py @@ -459,7 +459,7 @@ def check_python(supported_minors=[9, 10, 11, 12], reason=None): def check_diffusers(): if args.skip_all or args.skip_requirements: return - sha = 'c96bfa5c80eca798d555a79a491043c311d0f608' + sha = '63b631f38336f56755fb5cf15d9b0fb70bbf6323' # diffusers commit hash pkg = pkg_resources.working_set.by_key.get('diffusers', None) minor = int(pkg.version.split('.')[1] if pkg is not None else 0) cur = opts.get('diffusers_version', '') if minor > 0 else '' diff --git a/modules/processing_args.py b/modules/processing_args.py index 4ce552825..d73762d29 100644 --- a/modules/processing_args.py +++ b/modules/processing_args.py @@ -186,6 +186,21 @@ def set_pipeline_args(p, model, prompts: list, negative_prompts: list, prompts_2 shared.log.error(f'Sampler timesteps: {e}') else: shared.log.warning(f'Sampler: sampler={model.scheduler.__class__.__name__} timesteps not supported') + if 'sigmas' in possible: + sigmas = re.split(',| ', shared.opts.schedulers_timesteps) + sigmas = [float(x)/1000.0 for x in sigmas if x.isdigit()] + if len(sigmas) > 0: + if hasattr(model.scheduler, 'set_timesteps') and "sigmas" in set(inspect.signature(model.scheduler.set_timesteps).parameters.keys()): + try: + args['sigmas'] = sigmas + p.steps = len(sigmas) + p.timesteps = sigmas + steps = p.steps + shared.log.debug(f'Sampler: steps={len(sigmas)} sigmas={sigmas}') + except Exception as e: + shared.log.error(f'Sampler sigmas: {e}') + else: + shared.log.warning(f'Sampler: sampler={model.scheduler.__class__.__name__} sigmas not supported') if hasattr(model, 'scheduler') and hasattr(model.scheduler, 'noise_sampler_seed') and hasattr(model.scheduler, 'noise_sampler'): model.scheduler.noise_sampler = None # noise needs to be reset instead of using cached values From 6c74fff223e6ba204cfcabaf0a1ec669d085c90b Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 3 Dec 2024 09:46:23 -0500 Subject: [PATCH 055/249] add nvml charts Signed-off-by: Vladimir Mandic --- extensions-builtin/sd-extension-system-info | 2 +- javascript/nvml.js | 32 +++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/extensions-builtin/sd-extension-system-info b/extensions-builtin/sd-extension-system-info index 6a2a28a4f..dfa01ce99 160000 --- a/extensions-builtin/sd-extension-system-info +++ b/extensions-builtin/sd-extension-system-info @@ -1 +1 @@ -Subproject commit 6a2a28a4f674b85e09824384ad842b801426b491 +Subproject commit dfa01ce99a17d76b45284ef28cef018ff52ac353 diff --git a/javascript/nvml.js b/javascript/nvml.js index cf0187367..39850c9d8 100644 --- a/javascript/nvml.js +++ b/javascript/nvml.js @@ -1,6 +1,32 @@ let nvmlInterval = null; // eslint-disable-line prefer-const let nvmlEl = null; let nvmlTable = null; +const chartData = { mem: [], load: [] }; + +async function updateNVMLChart(mem, load) { + const maxLen = 120; + const colorRangeMap = $.range_map({ + '0:5': '#fffafa', + '6:10': '#fff7ed', + '11:20': '#fed7aa', + '21:30': '#fdba74', + '31:40': '#fb923c', + '41:50': '#f97316', + '51:60': '#ea580c', + '61:70': '#c2410c', + '71:80': '#9a3412', + '81:90': '#7c2d12', + '91:100': '#6c2e12', + }); + const sparklineConfigLOAD = { type: 'bar', height: '100px', barWidth: '2px', barSpacing: '1px', chartRangeMin: 0, chartRangeMax: 100, barColor: '#89007D' }; + const sparklineConfigMEM = { type: 'bar', height: '100px', barWidth: '2px', barSpacing: '1px', chartRangeMin: 0, chartRangeMax: 100, colorMap: colorRangeMap, composite: true }; + if (chartData.load.length > maxLen) chartData.load.shift(); + chartData.load.push(load); + if (chartData.mem.length > maxLen) chartData.mem.shift(); + chartData.mem.push(mem); + $('#nvmlChart').sparkline(chartData.load, sparklineConfigLOAD); + $('#nvmlChart').sparkline(chartData.mem, sparklineConfigMEM); +} async function updateNVML() { try { @@ -35,6 +61,9 @@ async function updateNVML() { State${gpu.state} `; nvmlTbody.innerHTML = rows; + const mem = 100 * (gpu.memory?.used || 0) / (gpu.memory?.total || 1); + const load = 100 * (gpu.clock?.gpu?.[0] || 0) / (gpu.clock?.gpu?.[1] || 1); + updateNVMLChart(mem, load); } nvmlEl.style.display = 'block'; } catch (e) { @@ -56,7 +85,10 @@ async function initNVML() { `; + const nvmlChart = document.createElement('div'); + nvmlChart.id = 'nvmlChart'; nvmlEl.appendChild(nvmlTable); + nvmlEl.appendChild(nvmlChart); gradioApp().appendChild(nvmlEl); log('initNVML'); } From e928dcf2a300f807bd24499963ae45ab5a7b04e3 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 3 Dec 2024 09:52:15 -0500 Subject: [PATCH 056/249] cleanup nvml Signed-off-by: Vladimir Mandic --- javascript/nvml.js | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/javascript/nvml.js b/javascript/nvml.js index 39850c9d8..0a82cba1b 100644 --- a/javascript/nvml.js +++ b/javascript/nvml.js @@ -5,7 +5,7 @@ const chartData = { mem: [], load: [] }; async function updateNVMLChart(mem, load) { const maxLen = 120; - const colorRangeMap = $.range_map({ + const colorRangeMap = $.range_map({ // eslint-disable-line no-undef '0:5': '#fffafa', '6:10': '#fff7ed', '11:20': '#fed7aa', @@ -24,8 +24,8 @@ async function updateNVMLChart(mem, load) { chartData.load.push(load); if (chartData.mem.length > maxLen) chartData.mem.shift(); chartData.mem.push(mem); - $('#nvmlChart').sparkline(chartData.load, sparklineConfigLOAD); - $('#nvmlChart').sparkline(chartData.mem, sparklineConfigMEM); + $('#nvmlChart').sparkline(chartData.load, sparklineConfigLOAD); // eslint-disable-line no-undef + $('#nvmlChart').sparkline(chartData.mem, sparklineConfigMEM); // eslint-disable-line no-undef } async function updateNVML() { @@ -61,9 +61,7 @@ async function updateNVML() { State${gpu.state} `; nvmlTbody.innerHTML = rows; - const mem = 100 * (gpu.memory?.used || 0) / (gpu.memory?.total || 1); - const load = 100 * (gpu.clock?.gpu?.[0] || 0) / (gpu.clock?.gpu?.[1] || 1); - updateNVMLChart(mem, load); + updateNVMLChart(gpu.load.memory, gpu.load.gpu); } nvmlEl.style.display = 'block'; } catch (e) { From d90061bc6f37dfc828e5c1aea1423657d84ff941 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 4 Dec 2024 08:47:22 -0500 Subject: [PATCH 057/249] add offload warning Signed-off-by: Vladimir Mandic --- modules/sd_models.py | 17 +++++++++++------ wiki | 2 +- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/modules/sd_models.py b/modules/sd_models.py index 37567962c..101ff837b 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -34,6 +34,7 @@ debug_process = shared.log.trace if os.environ.get('SD_PROCESS_DEBUG', None) is not None else lambda *args, **kwargs: None diffusers_version = int(diffusers.__version__.split('.')[1]) checkpoint_tiles = checkpoint_titles # legacy compatibility +should_offload = ['sc', 'sd3', 'f1', 'hunyuandit', 'auraflow', 'omnigen'] class NoWatermark: @@ -320,11 +321,15 @@ def set_diffuser_offload(sd_model, op: str = 'model'): return if not (hasattr(sd_model, "has_accelerate") and sd_model.has_accelerate): sd_model.has_accelerate = False - if hasattr(sd_model, 'maybe_free_model_hooks') and shared.opts.diffusers_offload_mode == "none": - shared.log.debug(f'Setting {op}: offload={shared.opts.diffusers_offload_mode} limit={shared.opts.cuda_mem_fraction}') - sd_model.maybe_free_model_hooks() - sd_model.has_accelerate = False - if hasattr(sd_model, "enable_model_cpu_offload") and shared.opts.diffusers_offload_mode == "model": + if shared.opts.diffusers_offload_mode == "none": + if shared.sd_model_type in should_offload: + shared.log.warning(f'Setting {op}: offload={shared.opts.diffusers_offload_mode} type={shared.sd_model.__class__.__name__} large model') + else: + shared.log.debug(f'Setting {op}: offload={shared.opts.diffusers_offload_mode} limit={shared.opts.cuda_mem_fraction}') + if hasattr(sd_model, 'maybe_free_model_hooks'): + sd_model.maybe_free_model_hooks() + sd_model.has_accelerate = False + if shared.opts.diffusers_offload_mode == "model" and hasattr(sd_model, "enable_model_cpu_offload"): try: shared.log.debug(f'Setting {op}: offload={shared.opts.diffusers_offload_mode} limit={shared.opts.cuda_mem_fraction}') if shared.opts.diffusers_move_base or shared.opts.diffusers_move_unet or shared.opts.diffusers_move_refiner: @@ -339,7 +344,7 @@ def set_diffuser_offload(sd_model, op: str = 'model'): set_accelerate(sd_model) except Exception as e: shared.log.error(f'Setting {op}: offload={shared.opts.diffusers_offload_mode} {e}') - if hasattr(sd_model, "enable_sequential_cpu_offload") and shared.opts.diffusers_offload_mode == "sequential": + if shared.opts.diffusers_offload_mode == "sequential" and hasattr(sd_model, "enable_sequential_cpu_offload"): try: shared.log.debug(f'Setting {op}: offload={shared.opts.diffusers_offload_mode} limit={shared.opts.cuda_mem_fraction}') if shared.opts.diffusers_move_base or shared.opts.diffusers_move_unet or shared.opts.diffusers_move_refiner: diff --git a/wiki b/wiki index f57cdb49d..2a83f725b 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit f57cdb49d8ca928024b43525897d1c1379eab4c4 +Subproject commit 2a83f725bda6a81399f579ba7102741f71b0be39 From 620539c870c430d28e6b5dd9dbf07977ad4f3910 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 4 Dec 2024 09:25:57 -0500 Subject: [PATCH 058/249] lora maintain device Signed-off-by: Vladimir Mandic --- modules/lora/networks.py | 63 ++++++++++++++++-------------- modules/model_stablecascade.py | 6 +-- modules/processing_diffusers.py | 12 ++---- modules/processing_info.py | 1 + modules/prompt_parser_diffusers.py | 4 -- modules/sd_models.py | 2 + 6 files changed, 42 insertions(+), 46 deletions(-) diff --git a/modules/lora/networks.py b/modules/lora/networks.py index 14fce760a..735c00c4c 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -307,7 +307,7 @@ def network_load(names, te_multipliers=None, unet_multipliers=None, dyn_dims=Non # section: process loaded networks -def network_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv], weight, network_layer_name, wanted_names): +def network_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv], weight: torch.Tensor, network_layer_name: str, wanted_names: tuple): global bnb # pylint: disable=W0603 backup_size = 0 if len(loaded_networks) > 0 and network_layer_name is not None and any([net.modules.get(network_layer_name, None) for net in loaded_networks]): # noqa: C419 # pylint: disable=R1729 @@ -356,7 +356,7 @@ def network_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.n return backup_size -def network_calc_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv], weight, network_layer_name): +def network_calc_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv], weight: torch.Tensor, network_layer_name: str): if shared.opts.diffusers_offload_mode == "none": self.to(devices.device, non_blocking=True) batch_updown = None @@ -403,7 +403,7 @@ def network_calc_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn. return batch_updown, batch_ex_bias -def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv], updown, ex_bias): +def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv], updown: torch.Tensor, ex_bias: torch.Tensor, orig_device: torch.device): t0 = time.time() weights_backup = getattr(self, "network_weights_backup", None) bias_backup = getattr(self, "network_bias_backup", None) @@ -421,10 +421,10 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn if getattr(self, "quant_type", None) in ['nf4', 'fp4'] and bnb is not None: self.weight = bnb.nn.Params4bit(new_weight, quant_state=self.quant_state, quant_type=self.quant_type, blocksize=self.blocksize) else: - self.weight = torch.nn.Parameter(new_weight, requires_grad=False) + self.weight = torch.nn.Parameter(new_weight.to(device=orig_device, non_blocking=True), requires_grad=False) del new_weight else: - self.weight = torch.nn.Parameter(weights_backup, requires_grad=False) + self.weight = torch.nn.Parameter(weights_backup.to(device=orig_device, non_blocking=True), requires_grad=False) if hasattr(self, "qweight") and hasattr(self, "freeze"): self.freeze() if bias_backup is not None: @@ -434,10 +434,10 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn self.bias = None if ex_bias is not None: new_weight = bias_backup.to(devices.device, non_blocking=True) + ex_bias.to(devices.device, non_blocking=True) - self.bias = torch.nn.Parameter(new_weight, requires_grad=False) + self.bias = torch.nn.Parameter(new_weight.to(device=orig_device, non_blocking=True), requires_grad=False) del new_weight else: - self.bias = torch.nn.Parameter(bias_backup, requires_grad=False) + self.bias = torch.nn.Parameter(bias_backup.to(device=orig_device, non_blocking=True), requires_grad=False) else: self.bias = None t1 = time.time() @@ -457,14 +457,15 @@ def network_activate(): if shared.opts.diffusers_offload_mode == "sequential": sd_models.disable_offload(sd_model) sd_models.move_model(sd_model, device=devices.cpu) - modules = [] + modules = {} for component_name in ['text_encoder','text_encoder_2', 'unet', 'transformer']: component = getattr(sd_model, component_name, None) if component is not None and hasattr(component, 'named_modules'): - modules += list(component.named_modules()) + modules[component_name] = list(component.named_modules()) + total = sum(len(x) for x in modules.values()) if len(loaded_networks) > 0: pbar = rp.Progress(rp.TextColumn('[cyan]Apply network: type=LoRA'), rp.BarColumn(), rp.TaskProgressColumn(), rp.TimeRemainingColumn(), rp.TimeElapsedColumn(), rp.TextColumn('[cyan]{task.description}'), console=shared.console) - task = pbar.add_task(description='' , total=len(modules)) + task = pbar.add_task(description='' , total=total) else: task = None pbar = nullcontext() @@ -474,29 +475,31 @@ def network_activate(): backup_size = 0 weights_devices = [] weights_dtypes = [] - for _, module in modules: - network_layer_name = getattr(module, 'network_layer_name', None) - current_names = getattr(module, "network_current_names", ()) - if shared.state.interrupted or network_layer_name is None or current_names == wanted_names: + for component in modules.keys(): + orig_device = getattr(sd_model, component, None).device + for _, module in modules[component]: + network_layer_name = getattr(module, 'network_layer_name', None) + current_names = getattr(module, "network_current_names", ()) + if shared.state.interrupted or network_layer_name is None or current_names == wanted_names: + if task is not None: + pbar.update(task, advance=1, description=f'networks={len(loaded_networks)} skip') + continue + weight = getattr(module, 'weight', None) + weight = weight.to(devices.device, non_blocking=True) if weight is not None else None + backup_size += network_backup_weights(module, weight, network_layer_name, wanted_names) + batch_updown, batch_ex_bias = network_calc_weights(module, weight, network_layer_name) + weights_device, weights_dtype = network_apply_weights(module, batch_updown, batch_ex_bias, orig_device) + weights_devices.append(weights_device) + weights_dtypes.append(weights_dtype) + if batch_updown is not None or batch_ex_bias is not None: + applied += 1 + del weight, batch_updown, batch_ex_bias + module.network_current_names = wanted_names if task is not None: - pbar.update(task, advance=1, description=f'networks={len(loaded_networks)} skip') - continue - weight = getattr(module, 'weight', None) - weight = weight.to(devices.device, non_blocking=True) if weight is not None else None - backup_size += network_backup_weights(module, weight, network_layer_name, wanted_names) - batch_updown, batch_ex_bias = network_calc_weights(module, weight, network_layer_name) - weights_device, weights_dtype = network_apply_weights(module, batch_updown, batch_ex_bias) - weights_devices.append(weights_device) - weights_dtypes.append(weights_dtype) - if batch_updown is not None or batch_ex_bias is not None: - applied += 1 - del weight, batch_updown, batch_ex_bias - module.network_current_names = wanted_names - if task is not None: - pbar.update(task, advance=1, description=f'networks={len(loaded_networks)} modules={len(modules)} apply={applied} backup={backup_size}') + pbar.update(task, advance=1, description=f'networks={len(loaded_networks)} modules={len(modules)} apply={applied} backup={backup_size}') weights_devices, weights_dtypes = list(set([x for x in weights_devices if x is not None])), list(set([x for x in weights_dtypes if x is not None])) # noqa: C403 # pylint: disable=R1718 if debug and len(loaded_networks) > 0: - shared.log.debug(f'Load network: type=LoRA networks={len(loaded_networks)} modules={len(modules)} apply={applied} device={weights_devices} dtype={weights_dtypes} backup={backup_size} fuse={shared.opts.lora_fuse_diffusers} time={get_timers()}') + shared.log.debug(f'Load network: type=LoRA networks={len(loaded_networks)} modules={total} apply={applied} device={weights_devices} dtype={weights_dtypes} backup={backup_size} fuse={shared.opts.lora_fuse_diffusers} time={get_timers()}') modules.clear() if shared.opts.diffusers_offload_mode == "sequential": sd_models.set_diffuser_offload(sd_model, op="model") diff --git a/modules/model_stablecascade.py b/modules/model_stablecascade.py index d6f9e4266..2a7739e55 100644 --- a/modules/model_stablecascade.py +++ b/modules/model_stablecascade.py @@ -187,8 +187,7 @@ def __call__( callback_on_step_end=None, callback_on_step_end_tensor_inputs=["latents"], ): - if shared.opts.diffusers_offload_mode == "balanced": - shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) # 0. Define commonly used variables self.guidance_scale = guidance_scale self.do_classifier_free_guidance = self.guidance_scale > 1 @@ -330,8 +329,7 @@ def __call__( elif output_type == "pil": images = images.permute(0, 2, 3, 1).cpu().float().numpy() # float() as bfloat16-> numpy doesnt work images = self.numpy_to_pil(images) - if shared.opts.diffusers_offload_mode == "balanced": - shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) else: images = latents diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index c605a761c..0341cac4d 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -83,8 +83,7 @@ def process_base(p: processing.StableDiffusionProcessing): try: t0 = time.time() sd_models_compile.check_deepcache(enable=True) - if shared.opts.diffusers_offload_mode == "balanced": - shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) sd_models.move_model(shared.sd_model, devices.device) if hasattr(shared.sd_model, 'unet'): sd_models.move_model(shared.sd_model.unet, devices.device) @@ -266,8 +265,7 @@ def process_refine(p: processing.StableDiffusionProcessing, output): if shared.state.interrupted or shared.state.skipped: shared.sd_model = orig_pipeline return output - if shared.opts.diffusers_offload_mode == "balanced": - shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) if shared.opts.diffusers_move_refiner: sd_models.move_model(shared.sd_refiner, devices.device) if hasattr(shared.sd_refiner, 'unet'): @@ -407,8 +405,7 @@ def process_diffusers(p: processing.StableDiffusionProcessing): shared.sd_model = orig_pipeline return results - if shared.opts.diffusers_offload_mode == "balanced": - shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) # sanitize init_images if hasattr(p, 'init_images') and getattr(p, 'init_images', None) is None: @@ -463,8 +460,7 @@ def process_diffusers(p: processing.StableDiffusionProcessing): shared.sd_model = orig_pipeline - if shared.opts.diffusers_offload_mode == "balanced": - shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) if p.state == '': global last_p # pylint: disable=global-statement diff --git a/modules/processing_info.py b/modules/processing_info.py index 714ebf35f..e0fca12ae 100644 --- a/modules/processing_info.py +++ b/modules/processing_info.py @@ -140,6 +140,7 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts=None, all_seeds=No if sd_hijack is not None and hasattr(sd_hijack.model_hijack, 'embedding_db') and len(sd_hijack.model_hijack.embedding_db.embeddings_used) > 0: # this is for original hijaacked models only, diffusers are handled separately args["Embeddings"] = ', '.join(sd_hijack.model_hijack.embedding_db.embeddings_used) # samplers + if getattr(p, 'sampler_name', None) is not None: args["Sampler eta delta"] = shared.opts.eta_noise_seed_delta if shared.opts.eta_noise_seed_delta != 0 and sd_samplers_common.is_sampler_using_eta_noise_seed_delta(p) else None args["Sampler eta multiplier"] = p.initial_noise_multiplier if getattr(p, 'initial_noise_multiplier', 1.0) != 1.0 else None diff --git a/modules/prompt_parser_diffusers.py b/modules/prompt_parser_diffusers.py index d2093351a..06c0b6012 100644 --- a/modules/prompt_parser_diffusers.py +++ b/modules/prompt_parser_diffusers.py @@ -39,8 +39,6 @@ def prepare_model(pipe = None): pipe = pipe.pipe if not hasattr(pipe, "text_encoder"): return None - # if shared.opts.diffusers_offload_mode == "balanced": - # pipe = sd_models.apply_balanced_offload(pipe) elif hasattr(pipe, "maybe_free_model_hooks"): pipe.maybe_free_model_hooks() devices.torch_gc() @@ -79,8 +77,6 @@ def __init__(self, prompts, negative_prompts, steps, clip_skip, p): self.scheduled_encode(pipe, batchidx) else: self.encode(pipe, prompt, negative_prompt, batchidx) - # if shared.opts.diffusers_offload_mode == "balanced": - # pipe = sd_models.apply_balanced_offload(pipe) self.checkcache(p) debug(f"Prompt encode: time={(time.time() - t0):.3f}") diff --git a/modules/sd_models.py b/modules/sd_models.py index 101ff837b..63ec6b327 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -420,6 +420,8 @@ def detach_hook(self, module): def apply_balanced_offload(sd_model): global offload_hook_instance # pylint: disable=global-statement + if shared.opts.diffusers_offload_mode != "balanced": + return sd_model if offload_hook_instance is None or offload_hook_instance.min_watermark != shared.opts.diffusers_offload_min_gpu_memory or offload_hook_instance.max_watermark != shared.opts.diffusers_offload_max_gpu_memory: offload_hook_instance = OffloadHook() t0 = time.time() From bf89599985a6a57815daf8ce40007ff972c11ae1 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Thu, 5 Dec 2024 11:42:24 +0300 Subject: [PATCH 059/249] pin ultralytics --- modules/postprocess/yolo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/postprocess/yolo.py b/modules/postprocess/yolo.py index 5deab1282..7bc259391 100644 --- a/modules/postprocess/yolo.py +++ b/modules/postprocess/yolo.py @@ -63,7 +63,7 @@ def enumerate(self): def dependencies(self): import installer - installer.install('ultralytics', ignore=True, quiet=True) + installer.install('ultralytics==8.3.40', ignore=True, quiet=True) def predict( self, From b5885311503690c7dc38b6f5757217beb15124c6 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Thu, 5 Dec 2024 11:43:49 +0300 Subject: [PATCH 060/249] pin ultralytics --- installer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installer.py b/installer.py index e03512301..c1ec1f177 100644 --- a/installer.py +++ b/installer.py @@ -1002,7 +1002,7 @@ def install_optional(): install('optimum-quanto', ignore=True) install('bitsandbytes', ignore=True) install('pynvml', ignore=True) - install('ultralytics', ignore=True) + install('ultralytics==8.3.40', ignore=True) install('Cython', ignore=True) install('insightface', ignore=True) # problematic build install('nncf==2.7.0', ignore=True, no_deps=True) # requires older pandas From 353431ddbd09e5b715a91736b8c671571b2d1bb8 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 5 Dec 2024 07:58:52 -0500 Subject: [PATCH 061/249] change offload and upcast defaults Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 12 +++++++++--- configs/flux/vae/config.json | 2 +- configs/sd15/vae/config.json | 1 + configs/sd3/vae/config.json | 2 +- configs/sdxl/vae/config.json | 2 +- modules/shared.py | 14 +++++++------- 6 files changed, 20 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 51d61aa46..d49c1c555 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,11 +47,17 @@ - Flux: all-in-one safetensors example: - Flux: do not recast quants -- **Offload** improvements: - - faster and more compatible *balanced* mode +- **Memory** improvements: + - faster and more compatible *balanced offload* mode - balanced offload: units are now in percentage instead of bytes - balanced offload: add both high and low watermark - *note*: balanced offload is recommended method for offload when using any large models such as sd35 or flux + default is 25% for low-watermark (skip offload if memory usage is below 25%) and 70% high-watermark (must offload if memory usage is above 70%) + - change-in-behavior: + `lowvrwam` triggers *sequential offload*, also automatically triggered on systems with <=4GB vram + all other systems use *balanced offload* by default (can be changed in settings) + previous behavior was to use *model offload* on systems with <=8GB and `medvram` and no offload by default + - VAE upcase is now disabled by default on all systems + if you have issues with image decode, you'll need to enable it manually - **UI**: - improved stats on generate completion - improved live preview display and performance diff --git a/configs/flux/vae/config.json b/configs/flux/vae/config.json index b43183d0f..7ecb342c2 100644 --- a/configs/flux/vae/config.json +++ b/configs/flux/vae/config.json @@ -14,7 +14,7 @@ "DownEncoderBlock2D", "DownEncoderBlock2D" ], - "force_upcast": true, + "force_upcast": false, "in_channels": 3, "latent_channels": 16, "latents_mean": null, diff --git a/configs/sd15/vae/config.json b/configs/sd15/vae/config.json index 55d78924f..2cba0e824 100644 --- a/configs/sd15/vae/config.json +++ b/configs/sd15/vae/config.json @@ -14,6 +14,7 @@ "DownEncoderBlock2D", "DownEncoderBlock2D" ], + "force_upcast": false, "in_channels": 3, "latent_channels": 4, "layers_per_block": 2, diff --git a/configs/sd3/vae/config.json b/configs/sd3/vae/config.json index 58e7764fb..f6f4e8684 100644 --- a/configs/sd3/vae/config.json +++ b/configs/sd3/vae/config.json @@ -15,7 +15,7 @@ "DownEncoderBlock2D", "DownEncoderBlock2D" ], - "force_upcast": true, + "force_upcast": false, "in_channels": 3, "latent_channels": 16, "latents_mean": null, diff --git a/configs/sdxl/vae/config.json b/configs/sdxl/vae/config.json index a66a171ba..1c7a60866 100644 --- a/configs/sdxl/vae/config.json +++ b/configs/sdxl/vae/config.json @@ -15,7 +15,7 @@ "DownEncoderBlock2D", "DownEncoderBlock2D" ], - "force_upcast": true, + "force_upcast": false, "in_channels": 3, "latent_channels": 4, "layers_per_block": 2, diff --git a/modules/shared.py b/modules/shared.py index 6b5c3f9f1..59e9e9ab7 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -432,15 +432,15 @@ def get_default_modes(): cmd_opts.lowvram = True default_offload_mode = "sequential" log.info(f"Device detect: memory={gpu_memory:.1f} optimization=lowvram") - elif gpu_memory <= 8: - cmd_opts.medvram = True - default_offload_mode = "model" - log.info(f"Device detect: memory={gpu_memory:.1f} optimization=medvram") + # elif gpu_memory <= 8: + # cmd_opts.medvram = True + # default_offload_mode = "model" + # log.info(f"Device detect: memory={gpu_memory:.1f} optimization=medvram") else: - default_offload_mode = "none" - log.info(f"Device detect: memory={gpu_memory:.1f} optimization=none") + default_offload_mode = "balanced" + log.info(f"Device detect: memory={gpu_memory:.1f} optimization=balanced") elif cmd_opts.medvram: - default_offload_mode = "model" + default_offload_mode = "balanced" elif cmd_opts.lowvram: default_offload_mode = "sequential" From 73c060297e5193828ce8a51466d63ed033a4d69b Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 5 Dec 2024 09:04:42 -0500 Subject: [PATCH 062/249] lora one more safe cast Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 4 ++-- modules/lora/networks.py | 8 ++------ modules/sd_detect.py | 2 +- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d49c1c555..80c5dd474 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Change Log for SD.Next -## Update for 2024-12-02 +## Update for 2024-12-05 ### New models and integrations @@ -53,7 +53,7 @@ - balanced offload: add both high and low watermark default is 25% for low-watermark (skip offload if memory usage is below 25%) and 70% high-watermark (must offload if memory usage is above 70%) - change-in-behavior: - `lowvrwam` triggers *sequential offload*, also automatically triggered on systems with <=4GB vram + low-end systems, triggered by either `lowvrwam` or by detection of <=4GB will use *sequential offload* all other systems use *balanced offload* by default (can be changed in settings) previous behavior was to use *model offload* on systems with <=8GB and `medvram` and no offload by default - VAE upcase is now disabled by default on all systems diff --git a/modules/lora/networks.py b/modules/lora/networks.py index 735c00c4c..20626d9ef 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -369,11 +369,11 @@ def network_calc_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn. updown, ex_bias = module.calc_updown(weight) t1 = time.time() if batch_updown is not None and updown is not None: - batch_updown += updown + batch_updown += updown.to(batch_updown.device, non_blocking=True) else: batch_updown = updown if batch_ex_bias is not None and ex_bias is not None: - batch_ex_bias += ex_bias + batch_ex_bias += ex_bias.to(batch_ex_bias.device, non_blocking=True) else: batch_ex_bias = ex_bias timer['calc'] += t1 - t0 @@ -396,10 +396,6 @@ def network_calc_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn. errors.display(e, 'LoRA') raise RuntimeError('LoRA apply weight') from e continue - if module is None: - continue - shared.log.warning(f'LoRA network="{net.name}" layer="{network_layer_name}" unsupported operation') - extra_network_lora.errors[net.name] = extra_network_lora.errors.get(net.name, 0) + 1 return batch_updown, batch_ex_bias diff --git a/modules/sd_detect.py b/modules/sd_detect.py index 062bb32e1..071a83d7e 100644 --- a/modules/sd_detect.py +++ b/modules/sd_detect.py @@ -105,7 +105,7 @@ def detect_pipeline(f: str, op: str = 'model', warning=True, quiet=False): guess = 'Stable Diffusion XL Instruct' # get actual pipeline pipeline = shared_items.get_pipelines().get(guess, None) if pipeline is None else pipeline - if not quiet: + if debug_load is not None: shared.log.info(f'Autodetect {op}: detect="{guess}" class={getattr(pipeline, "__name__", None)} file="{f}" size={size}MB') t0 = time.time() keys = model_tools.get_safetensor_keys(f) From 56da8109dbae6c7f38d894e208472eb58426f5cb Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 5 Dec 2024 10:03:17 -0500 Subject: [PATCH 063/249] flux redux allow prompt Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 1 + installer.py | 2 +- scripts/flux_tools.py | 39 ++++++++++++++++++++++++++++++++------- 3 files changed, 34 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 80c5dd474..601cba683 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ *todo*: support for Canny/Depth LoRAs - [Redux](https://huggingface.co/black-forest-labs/FLUX.1-Redux-dev): ~0.1GB works together with existing model and basically uses input image to analyze it and use that instead of prompt + *optional* can use prompt to combine guidance with input image *recommended*: low denoise strength levels result in more variety - [Fill](https://huggingface.co/black-forest-labs/FLUX.1-Fill-dev): ~23.8GB, replaces currently loaded model *note*: can be used in inpaint/outpaint mode only diff --git a/installer.py b/installer.py index c1ec1f177..93cd10413 100644 --- a/installer.py +++ b/installer.py @@ -459,7 +459,7 @@ def check_python(supported_minors=[9, 10, 11, 12], reason=None): def check_diffusers(): if args.skip_all or args.skip_requirements: return - sha = '63b631f38336f56755fb5cf15d9b0fb70bbf6323' # diffusers commit hash + sha = '3335e2262d47e7d7e311a44dea7f454b5f01b643' # diffusers commit hash pkg = pkg_resources.working_set.by_key.get('diffusers', None) minor = int(pkg.version.split('.')[1] if pkg is not None else 0) cur = opts.get('diffusers_version', '') if minor > 0 else '' diff --git a/scripts/flux_tools.py b/scripts/flux_tools.py index 3fbab6c6f..909257a37 100644 --- a/scripts/flux_tools.py +++ b/scripts/flux_tools.py @@ -26,11 +26,21 @@ def ui(self, _is_img2img): # ui elements with gr.Row(): tool = gr.Dropdown(label='Tool', choices=['None', 'Redux', 'Fill', 'Canny', 'Depth'], value='None') with gr.Row(): - process = gr.Checkbox(label='Preprocess input images', value=True) - strength = gr.Checkbox(label='Override denoise strength', value=True) - return [tool, strength, process] + prompt = gr.Slider(label='Redux prompt strength', minimum=0, maximum=2, step=0.01, value=0, visible=False) + process = gr.Checkbox(label='Control preprocess input images', value=True, visible=False) + strength = gr.Checkbox(label='Control override denoise strength', value=True, visible=False) - def run(self, p: processing.StableDiffusionProcessing, tool: str = 'None', strength: bool = True, process: bool = True): # pylint: disable=arguments-differ + def display(tool): + return [ + gr.update(visible=tool in ['Redux']), + gr.update(visible=tool in ['Canny', 'Depth']), + gr.update(visible=tool in ['Canny', 'Depth']), + ] + + tool.change(fn=display, inputs=[tool], outputs=[prompt, process, strength]) + return [tool, prompt, strength, process] + + def run(self, p: processing.StableDiffusionProcessing, tool: str = 'None', prompt: float = 1.0, strength: bool = True, process: bool = True): # pylint: disable=arguments-differ global redux_pipe, processor_canny, processor_depth # pylint: disable=global-statement if tool is None or tool == 'None': return @@ -50,6 +60,7 @@ def run(self, p: processing.StableDiffusionProcessing, tool: str = 'None', stren t0 = time.time() if tool == 'Redux': # pipe_prior_redux = FluxPriorReduxPipeline.from_pretrained("black-forest-labs/FLUX.1-Redux-dev", revision="refs/pr/8", torch_dtype=torch.bfloat16).to("cuda") + shared.log.debug(f'{title}: tool={tool} prompt={prompt}') if redux_pipe is None: redux_pipe = diffusers.FluxPriorReduxPipeline.from_pretrained( "black-forest-labs/FLUX.1-Redux-dev", @@ -57,7 +68,21 @@ def run(self, p: processing.StableDiffusionProcessing, tool: str = 'None', stren torch_dtype=devices.dtype, cache_dir=shared.opts.hfcache_dir ).to(devices.device) - redux_output = redux_pipe(image) + if prompt > 0: + shared.log.info(f'{title}: tool={tool} load text encoder') + redux_pipe.tokenizer, redux_pipe.tokenizer_2 = shared.sd_model.tokenizer, shared.sd_model.tokenizer_2 + redux_pipe.text_encoder, redux_pipe.text_encoder_2 = shared.sd_model.text_encoder, shared.sd_model.text_encoder_2 + sd_models.apply_balanced_offload(redux_pipe) + redux_output = redux_pipe( + image=image, + prompt=p.prompt if prompt > 0 else None, + prompt_embeds_scale=[prompt], + pooled_prompt_embeds_scale=[prompt], + ) + if prompt > 0: + redux_pipe.tokenizer, redux_pipe.tokenizer_2 = None, None + redux_pipe.text_encoder, redux_pipe.text_encoder_2 = None, None + devices.torch_gc() for k, v in redux_output.items(): p.task_args[k] = v else: @@ -77,7 +102,7 @@ def run(self, p: processing.StableDiffusionProcessing, tool: str = 'None', stren p.task_args['mask_image'] = p.image_mask if tool == 'Canny': - # pipe = FluxControlPipeline.from_pretrained("black-forest-labs/FLUX.1-Canny-dev", torch_dtype=torch.bfloat16, revision="refs/pr/1").to("cuda") + # pipe = diffusers.FluxControlPipeline.from_pretrained("black-forest-labs/FLUX.1-Canny-dev", torch_dtype=torch.bfloat16, revision="refs/pr/1").to("cuda") install('controlnet-aux') install('timm==0.9.16') if shared.sd_model.__class__.__name__ != 'FluxControlPipeline' or 'Canny' not in shared.opts.sd_model_checkpoint: @@ -99,7 +124,7 @@ def run(self, p: processing.StableDiffusionProcessing, tool: str = 'None', stren processor_canny = None if tool == 'Depth': - # pipe = FluxControlPipeline.from_pretrained("black-forest-labs/FLUX.1-Depth-dev", torch_dtype=torch.bfloat16, revision="refs/pr/1").to("cuda") + # pipe = diffusers.FluxControlPipeline.from_pretrained("black-forest-labs/FLUX.1-Depth-dev", torch_dtype=torch.bfloat16, revision="refs/pr/1").to("cuda") install('git+https://github.com/huggingface/image_gen_aux.git', 'image_gen_aux') if shared.sd_model.__class__.__name__ != 'FluxControlPipeline' or 'Depth' not in shared.opts.sd_model_checkpoint: shared.opts.data["sd_model_checkpoint"] = "black-forest-labs/FLUX.1-Depth-dev" From 1d8922c7e127fbe14e04251a66f9c0827170119c Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 6 Dec 2024 07:23:31 -0500 Subject: [PATCH 064/249] safer lora unapply Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 2 +- modules/lora/networks.py | 19 ++++++++++--------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 601cba683..be9a573ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Change Log for SD.Next -## Update for 2024-12-05 +## Update for 2024-12-06 ### New models and integrations diff --git a/modules/lora/networks.py b/modules/lora/networks.py index 20626d9ef..f1fdb0c45 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -334,6 +334,7 @@ def network_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.n if shared.opts.lora_offload_backup and weights_backup is not None and isinstance(weights_backup, torch.Tensor): weights_backup = weights_backup.to(devices.cpu) self.network_weights_backup = weights_backup + bias_backup = getattr(self, "network_bias_backup", None) if bias_backup is None: if getattr(self, 'bias', None) is not None: @@ -380,12 +381,9 @@ def network_calc_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn. if shared.opts.diffusers_offload_mode != "none": t0 = time.time() if batch_updown is not None: - batch_updown = batch_updown.to(devices.cpu, non_blocking=True) + batch_updown = batch_updown.to(devices.cpu) if batch_ex_bias is not None: - batch_ex_bias = batch_ex_bias.to(devices.cpu, non_blocking=True) - if devices.backend == "ipex": - # using non_blocking=True here causes NaNs on Intel - torch.xpu.synchronize(devices.device) + batch_ex_bias = batch_ex_bias.to(devices.cpu) t1 = time.time() timer['move'] += t1 - t0 except RuntimeError as e: @@ -405,6 +403,7 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn bias_backup = getattr(self, "network_bias_backup", None) if weights_backup is None and bias_backup is None: return None, None + if weights_backup is not None: if isinstance(weights_backup, bool): weights_backup = self.weight @@ -417,12 +416,13 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn if getattr(self, "quant_type", None) in ['nf4', 'fp4'] and bnb is not None: self.weight = bnb.nn.Params4bit(new_weight, quant_state=self.quant_state, quant_type=self.quant_type, blocksize=self.blocksize) else: - self.weight = torch.nn.Parameter(new_weight.to(device=orig_device, non_blocking=True), requires_grad=False) + self.weight = torch.nn.Parameter(new_weight.to(device=orig_device), requires_grad=False) del new_weight else: - self.weight = torch.nn.Parameter(weights_backup.to(device=orig_device, non_blocking=True), requires_grad=False) + self.weight = torch.nn.Parameter(weights_backup.to(device=orig_device), requires_grad=False) if hasattr(self, "qweight") and hasattr(self, "freeze"): self.freeze() + if bias_backup is not None: if isinstance(bias_backup, bool): bias_backup = self.bias @@ -430,12 +430,13 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn self.bias = None if ex_bias is not None: new_weight = bias_backup.to(devices.device, non_blocking=True) + ex_bias.to(devices.device, non_blocking=True) - self.bias = torch.nn.Parameter(new_weight.to(device=orig_device, non_blocking=True), requires_grad=False) + self.bias = torch.nn.Parameter(new_weight.to(device=orig_device), requires_grad=False) del new_weight else: - self.bias = torch.nn.Parameter(bias_backup.to(device=orig_device, non_blocking=True), requires_grad=False) + self.bias = torch.nn.Parameter(bias_backup.to(device=orig_device), requires_grad=False) else: self.bias = None + t1 = time.time() timer['apply'] += t1 - t0 return self.weight.device, self.weight.dtype From 0c509e59442bf102bd167030535f777a99fec15b Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 6 Dec 2024 07:36:20 -0500 Subject: [PATCH 065/249] handle os err Signed-off-by: Vladimir Mandic --- modules/memstats.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/memstats.py b/modules/memstats.py index 7836f7636..fd5f152a0 100644 --- a/modules/memstats.py +++ b/modules/memstats.py @@ -20,10 +20,10 @@ def memory_stats(): mem.update({ 'ram': ram }) except Exception as e: if not fail_once: - shared.log.error('Memory stats: {e}') + shared.log.error(f'Memory stats: {e}') errors.display(e, 'Memory stats') fail_once = True - mem.update({ 'ram': str(e) }) + mem.update({ 'ram': { 'error': str(e) } }) try: s = torch.cuda.mem_get_info() gpu = { 'used': gb(s[1] - s[0]), 'total': gb(s[1]) } From 2303d558f4e63c07650959e65abe468f16bf040a Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 6 Dec 2024 09:15:37 -0500 Subject: [PATCH 066/249] remove non-blocking Signed-off-by: Vladimir Mandic --- cli/load-unet.py | 4 ++-- modules/lora/networks.py | 12 ++++++------ modules/processing_vae.py | 2 +- modules/rife/__init__.py | 4 ++-- modules/sd_hijack_accelerate.py | 8 ++++---- modules/sd_models.py | 2 +- 6 files changed, 16 insertions(+), 16 deletions(-) diff --git a/cli/load-unet.py b/cli/load-unet.py index 2398cdb64..c910101b0 100644 --- a/cli/load-unet.py +++ b/cli/load-unet.py @@ -33,13 +33,13 @@ def set_module_tensor( stats.dtypes[value.dtype] = 0 stats.dtypes[value.dtype] += 1 if name in module._buffers: # pylint: disable=protected-access - module._buffers[name] = value.to(device=device, dtype=dtype, non_blocking=True) # pylint: disable=protected-access + module._buffers[name] = value.to(device=device, dtype=dtype) # pylint: disable=protected-access if 'buffers' not in stats.weights: stats.weights['buffers'] = 0 stats.weights['buffers'] += 1 elif value is not None: param_cls = type(module._parameters[name]) # pylint: disable=protected-access - module._parameters[name] = param_cls(value, requires_grad=old_value.requires_grad).to(device, dtype=dtype, non_blocking=True) # pylint: disable=protected-access + module._parameters[name] = param_cls(value, requires_grad=old_value.requires_grad).to(device, dtype=dtype) # pylint: disable=protected-access if 'parameters' not in stats.weights: stats.weights['parameters'] = 0 stats.weights['parameters'] += 1 diff --git a/modules/lora/networks.py b/modules/lora/networks.py index f1fdb0c45..5a093370c 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -359,7 +359,7 @@ def network_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.n def network_calc_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv], weight: torch.Tensor, network_layer_name: str): if shared.opts.diffusers_offload_mode == "none": - self.to(devices.device, non_blocking=True) + self.to(devices.device) batch_updown = None batch_ex_bias = None for net in loaded_networks: @@ -370,11 +370,11 @@ def network_calc_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn. updown, ex_bias = module.calc_updown(weight) t1 = time.time() if batch_updown is not None and updown is not None: - batch_updown += updown.to(batch_updown.device, non_blocking=True) + batch_updown += updown.to(batch_updown.device) else: batch_updown = updown if batch_ex_bias is not None and ex_bias is not None: - batch_ex_bias += ex_bias.to(batch_ex_bias.device, non_blocking=True) + batch_ex_bias += ex_bias.to(batch_ex_bias.device) else: batch_ex_bias = ex_bias timer['calc'] += t1 - t0 @@ -412,7 +412,7 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn if updown is not None and len(weights_backup.shape) == 4 and weights_backup.shape[1] == 9: # inpainting model. zero pad updown to make channel[1] 4 to 9 updown = torch.nn.functional.pad(updown, (0, 0, 0, 0, 0, 5)) # pylint: disable=not-callable if updown is not None: - new_weight = weights_backup.to(devices.device, non_blocking=True) + updown.to(devices.device, non_blocking=True) + new_weight = weights_backup.to(devices.device) + updown.to(devices.device) if getattr(self, "quant_type", None) in ['nf4', 'fp4'] and bnb is not None: self.weight = bnb.nn.Params4bit(new_weight, quant_state=self.quant_state, quant_type=self.quant_type, blocksize=self.blocksize) else: @@ -429,7 +429,7 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn else: self.bias = None if ex_bias is not None: - new_weight = bias_backup.to(devices.device, non_blocking=True) + ex_bias.to(devices.device, non_blocking=True) + new_weight = bias_backup.to(devices.device) + ex_bias.to(devices.device) self.bias = torch.nn.Parameter(new_weight.to(device=orig_device), requires_grad=False) del new_weight else: @@ -482,7 +482,7 @@ def network_activate(): pbar.update(task, advance=1, description=f'networks={len(loaded_networks)} skip') continue weight = getattr(module, 'weight', None) - weight = weight.to(devices.device, non_blocking=True) if weight is not None else None + weight = weight.to(devices.device) if weight is not None else None backup_size += network_backup_weights(module, weight, network_layer_name, wanted_names) batch_updown, batch_ex_bias = network_calc_weights(module, weight, network_layer_name) weights_device, weights_dtype = network_apply_weights(module, batch_updown, batch_ex_bias, orig_device) diff --git a/modules/processing_vae.py b/modules/processing_vae.py index b114e01d3..1c4a45f07 100644 --- a/modules/processing_vae.py +++ b/modules/processing_vae.py @@ -117,7 +117,7 @@ def full_vae_decode(latents, model): model.vae.orig_dtype = model.vae.dtype model.vae = model.vae.to(dtype=torch.float32) latents = latents.to(torch.float32) - latents = latents.to(devices.device, non_blocking=True) + latents = latents.to(devices.device) if getattr(model.vae, "post_quant_conv", None) is not None: latents = latents.to(next(iter(model.vae.post_quant_conv.parameters())).dtype) diff --git a/modules/rife/__init__.py b/modules/rife/__init__.py index f74f3d984..2a636eb2f 100644 --- a/modules/rife/__init__.py +++ b/modules/rife/__init__.py @@ -82,13 +82,13 @@ def f_pad(img): for _i in range(pad): # fill starting frames buffer.put(frame) - I1 = f_pad(torch.from_numpy(np.transpose(frame, (2,0,1))).to(devices.device, non_blocking=True).unsqueeze(0).float() / 255.) + I1 = f_pad(torch.from_numpy(np.transpose(frame, (2,0,1))).to(devices.device).unsqueeze(0).float() / 255.) with torch.no_grad(): with tqdm(total=len(images), desc='Interpolate', unit='frame') as pbar: for image in images: frame = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR) I0 = I1 - I1 = f_pad(torch.from_numpy(np.transpose(frame, (2,0,1))).to(devices.device, non_blocking=True).unsqueeze(0).float() / 255.) + I1 = f_pad(torch.from_numpy(np.transpose(frame, (2,0,1))).to(devices.device).unsqueeze(0).float() / 255.) I0_small = F.interpolate(I0, (32, 32), mode='bilinear', align_corners=False).to(torch.float32) I1_small = F.interpolate(I1, (32, 32), mode='bilinear', align_corners=False).to(torch.float32) ssim = ssim_matlab(I0_small[:, :3], I1_small[:, :3]) diff --git a/modules/sd_hijack_accelerate.py b/modules/sd_hijack_accelerate.py index 90eac5c4e..f8cf8983f 100644 --- a/modules/sd_hijack_accelerate.py +++ b/modules/sd_hijack_accelerate.py @@ -35,10 +35,10 @@ def hijack_set_module_tensor( with devices.inference_context(): # note: majority of time is spent on .to(old_value.dtype) if tensor_name in module._buffers: # pylint: disable=protected-access - module._buffers[tensor_name] = value.to(device, old_value.dtype, non_blocking=True) # pylint: disable=protected-access + module._buffers[tensor_name] = value.to(device, old_value.dtype) # pylint: disable=protected-access elif value is not None or not devices.same_device(torch.device(device), module._parameters[tensor_name].device): # pylint: disable=protected-access param_cls = type(module._parameters[tensor_name]) # pylint: disable=protected-access - module._parameters[tensor_name] = param_cls(value, requires_grad=old_value.requires_grad).to(device, old_value.dtype, non_blocking=True) # pylint: disable=protected-access + module._parameters[tensor_name] = param_cls(value, requires_grad=old_value.requires_grad).to(device, old_value.dtype) # pylint: disable=protected-access t1 = time.time() tensor_to_timer += (t1 - t0) @@ -63,10 +63,10 @@ def hijack_set_module_tensor_simple( old_value = getattr(module, tensor_name) with devices.inference_context(): if tensor_name in module._buffers: # pylint: disable=protected-access - module._buffers[tensor_name] = value.to(device, non_blocking=True) # pylint: disable=protected-access + module._buffers[tensor_name] = value.to(device) # pylint: disable=protected-access elif value is not None or not devices.same_device(torch.device(device), module._parameters[tensor_name].device): # pylint: disable=protected-access param_cls = type(module._parameters[tensor_name]) # pylint: disable=protected-access - module._parameters[tensor_name] = param_cls(value, requires_grad=old_value.requires_grad).to(device, non_blocking=True) # pylint: disable=protected-access + module._parameters[tensor_name] = param_cls(value, requires_grad=old_value.requires_grad).to(device) # pylint: disable=protected-access t1 = time.time() tensor_to_timer += (t1 - t0) diff --git a/modules/sd_models.py b/modules/sd_models.py index 63ec6b327..8853916e4 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -529,7 +529,7 @@ def move_model(model, device=None, force=False): t0 = time.time() try: if hasattr(model, 'to'): - model.to(device, non_blocking=True) + model.to(device) if hasattr(model, "prior_pipe"): model.prior_pipe.to(device) except Exception as e0: From 113db65df1e1319bbb67ef9afa2d1d90a8339449 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 6 Dec 2024 13:51:25 -0500 Subject: [PATCH 067/249] update wiki Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 1 + wiki | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index be9a573ea..b3aaec73f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -72,6 +72,7 @@ ### Updates +- Additional Wiki content: Styles, Wildcards, etc. - **OpenVINO**: update to 2024.5.0 - **Sampler** improvements - Euler FlowMatch: add sigma methods (*karras/exponential/betas*) diff --git a/wiki b/wiki index 2a83f725b..c5d484397 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 2a83f725bda6a81399f579ba7102741f71b0be39 +Subproject commit c5d484397f7504fdea098d5e24c843a69c9fd2a2 From d8221849b9758a3f83d89a6353fe62bcb1684164 Mon Sep 17 00:00:00 2001 From: AI-Casanova <54461896+AI-Casanova@users.noreply.github.com> Date: Fri, 6 Dec 2024 22:54:08 -0600 Subject: [PATCH 068/249] lora low memory mode: switching requires manual model reload --- modules/lora/networks.py | 67 +++++++++++++++++++++++++++++++++++++--- modules/shared.py | 1 + 2 files changed, 64 insertions(+), 4 deletions(-) diff --git a/modules/lora/networks.py b/modules/lora/networks.py index 5a093370c..9618f01a9 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -316,7 +316,7 @@ def network_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.n weights_backup = getattr(self, "network_weights_backup", None) if weights_backup is None and wanted_names != (): # pylint: disable=C1803 self.network_weights_backup = None - if shared.opts.lora_fuse_diffusers: + if shared.opts.lora_fuse_diffusers or shared.opts.lora_low_memory: weights_backup = True elif getattr(weight, "quant_type", None) in ['nf4', 'fp4']: if bnb is None: @@ -338,7 +338,7 @@ def network_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.n bias_backup = getattr(self, "network_bias_backup", None) if bias_backup is None: if getattr(self, 'bias', None) is not None: - if shared.opts.lora_fuse_diffusers: + if shared.opts.lora_fuse_diffusers or shared.opts.lora_low_memory: bias_backup = True else: bias_backup = self.bias.clone() @@ -397,7 +397,7 @@ def network_calc_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn. return batch_updown, batch_ex_bias -def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv], updown: torch.Tensor, ex_bias: torch.Tensor, orig_device: torch.device): +def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv], updown: torch.Tensor, ex_bias: torch.Tensor, orig_device: torch.device, deactivate: bool = False): t0 = time.time() weights_backup = getattr(self, "network_weights_backup", None) bias_backup = getattr(self, "network_bias_backup", None) @@ -412,6 +412,8 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn if updown is not None and len(weights_backup.shape) == 4 and weights_backup.shape[1] == 9: # inpainting model. zero pad updown to make channel[1] 4 to 9 updown = torch.nn.functional.pad(updown, (0, 0, 0, 0, 0, 5)) # pylint: disable=not-callable if updown is not None: + if deactivate: + updown *= -1 new_weight = weights_backup.to(devices.device) + updown.to(devices.device) if getattr(self, "quant_type", None) in ['nf4', 'fp4'] and bnb is not None: self.weight = bnb.nn.Params4bit(new_weight, quant_state=self.quant_state, quant_type=self.quant_type, blocksize=self.blocksize) @@ -429,6 +431,8 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn else: self.bias = None if ex_bias is not None: + if deactivate: + ex_bias *= -1 new_weight = bias_backup.to(devices.device) + ex_bias.to(devices.device) self.bias = torch.nn.Parameter(new_weight.to(device=orig_device), requires_grad=False) del new_weight @@ -443,7 +447,62 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn def network_deactivate(): - pass + if not shared.opts.lora_low_memory: + return + timer['deactivate'] = 0 + t0 = time.time() + sd_model = getattr(shared.sd_model, "pipe", shared.sd_model) # wrapped model compatiblility + if shared.opts.diffusers_offload_mode == "sequential": + sd_models.disable_offload(sd_model) + sd_models.move_model(sd_model, device=devices.cpu) + modules = {} + for component_name in ['text_encoder', 'text_encoder_2', 'unet', 'transformer']: + component = getattr(sd_model, component_name, None) + if component is not None and hasattr(component, 'named_modules'): + modules[component_name] = list(component.named_modules()) + total = sum(len(x) for x in modules.values()) + if len(loaded_networks) > 0: + pbar = rp.Progress(rp.TextColumn('[cyan]Deactivate network: type=LoRA'), rp.BarColumn(), rp.TaskProgressColumn(), + rp.TimeRemainingColumn(), rp.TimeElapsedColumn(), rp.TextColumn('[cyan]{task.description}'), + console=shared.console) + task = pbar.add_task(description='', total=total) + else: + task = None + pbar = nullcontext() + with devices.inference_context(), pbar: + applied = 0 + weights_devices = [] + weights_dtypes = [] + for component in modules.keys(): + orig_device = getattr(sd_model, component, None).device + for _, module in modules[component]: + network_layer_name = getattr(module, 'network_layer_name', None) + if shared.state.interrupted or network_layer_name is None: + if task is not None: + pbar.update(task, advance=1, description=f'networks={len(loaded_networks)} skip') + continue + weight = getattr(module, 'weight', None) + weight = weight.to(devices.device) if weight is not None else None + batch_updown, batch_ex_bias = network_calc_weights(module, weight, network_layer_name) + weights_device, weights_dtype = network_apply_weights(module, batch_updown, batch_ex_bias, orig_device, deactivate=True) + weights_devices.append(weights_device) + weights_dtypes.append(weights_dtype) + if batch_updown is not None or batch_ex_bias is not None: + applied += 1 + del weight, batch_updown, batch_ex_bias + module.network_current_names = () + if task is not None: + pbar.update(task, advance=1, + description=f'networks={len(loaded_networks)} modules={len(modules)} deactivate={applied}') + weights_devices, weights_dtypes = list(set([x for x in weights_devices if x is not None])), list(set([x for x in weights_dtypes if x is not None])) # noqa: C403 # pylint: disable=R1718 + if debug and len(loaded_networks) > 0: + shared.log.debug( + f'Deactivate network: type=LoRA networks={len(loaded_networks)} modules={total} deactivate={applied} device={weights_devices} dtype={weights_dtypes} fuse={shared.opts.lora_fuse_diffusers} time={get_timers()}') + modules.clear() + if shared.opts.diffusers_offload_mode == "sequential": + sd_models.set_diffuser_offload(sd_model, op="model") + t1 = time.time() + timer['deactivate'] += t1 - t0 def network_activate(): timer['backup'] = 0 diff --git a/modules/shared.py b/modules/shared.py index 59e9e9ab7..83c1cc0b3 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -913,6 +913,7 @@ def get_default_modes(): "lora_apply_tags": OptionInfo(0, "LoRA auto-apply tags", gr.Slider, {"minimum": -1, "maximum": 32, "step": 1}), "lora_in_memory_limit": OptionInfo(0, "LoRA memory cache", gr.Slider, {"minimum": 0, "maximum": 24, "step": 1}), "lora_quant": OptionInfo("NF4","LoRA precision in quantized models", gr.Radio, {"choices": ["NF4", "FP4"]}), + "lora_low_memory": OptionInfo(False, "LoRA low memory mode"), })) options_templates.update(options_section((None, "Internal options"), { From e180114cfaee8cf5e4db82000a4e233ccab8456b Mon Sep 17 00:00:00 2001 From: QuantumSoul Date: Sat, 7 Dec 2024 14:13:16 +0100 Subject: [PATCH 069/249] Create mkdocs.yml --- mkdocs.yml | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 mkdocs.yml diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 000000000..7fdeb24db --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,49 @@ +site_name: SD.Next Documentation +site_url: https://vladmandic.github.io/automatic +repo_url: https://github.com/vladmandic/automatic +repo_name: vladmandic/automatic +docs_dir: wiki + +theme: + name: material + + features: + - navigation.footer + - navigation.instant + - navigation.instant.prefetch + - navigation.instant.progress + - navigation.tracking + + palette: + - media: "(prefers-color-scheme: light)" + scheme: default + toggle: + icon: material/weather-night + name: Switch to dark mode + primary: teal + accent: pink + + - media: "(prefers-color-scheme: dark)" + scheme: slate + toggle: + icon: material/weather-sunny + name: Switch to light mode + primary: green + accent: yellow + + logo: assets/favicon.svg + favicon: assets/favicon.svg + +extra: + social: + - icon: fontawesome/brands/discord + link: https://discord.gg/VjvR2tabEX + name: Discord + - icon: fontawesome/brands/youtube + link: https://www.youtube.com/@SDNext + name: Youtube + +markdown_extensions: + - admonition + - pymdownx.details + - pymdownx.superfences \ No newline at end of file From 2892e829dbdcf4e79f01fd58cdfeeb6048c0f967 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 9 Dec 2024 13:40:19 -0500 Subject: [PATCH 070/249] yet another lora refactor Signed-off-by: Vladimir Mandic --- extensions-builtin/Lora/network_lora.py | 3 - mkdocs.yml | 49 ----- modules/extra_networks.py | 16 +- modules/face/faceid.py | 10 +- modules/lora/extra_networks_lora.py | 8 +- modules/lora/lora_timers.py | 38 ++++ modules/lora/network_lora.py | 3 - modules/lora/networks.py | 240 ++++++++++++------------ modules/processing.py | 14 +- modules/processing_args.py | 1 + modules/processing_callbacks.py | 2 +- modules/processing_class.py | 1 + modules/processing_diffusers.py | 15 +- modules/prompt_parser_diffusers.py | 2 +- modules/shared.py | 5 +- wiki | 2 +- 16 files changed, 193 insertions(+), 216 deletions(-) delete mode 100644 mkdocs.yml create mode 100644 modules/lora/lora_timers.py diff --git a/extensions-builtin/Lora/network_lora.py b/extensions-builtin/Lora/network_lora.py index 5e6eaef6c..8ebda2e22 100644 --- a/extensions-builtin/Lora/network_lora.py +++ b/extensions-builtin/Lora/network_lora.py @@ -22,7 +22,6 @@ def __init__(self, net: network.Network, weights: network.NetworkWeights): self.dim = weights.w["lora_down.weight"].shape[0] def create_module(self, weights, key, none_ok=False): - from modules.shared import opts weight = weights.get(key) if weight is None and none_ok: return None @@ -49,8 +48,6 @@ def create_module(self, weights, key, none_ok=False): if weight.shape != module.weight.shape: weight = weight.reshape(module.weight.shape) module.weight.copy_(weight) - if opts.lora_load_gpu: - module = module.to(device=devices.device, dtype=devices.dtype) module.weight.requires_grad_(False) return module diff --git a/mkdocs.yml b/mkdocs.yml deleted file mode 100644 index 7fdeb24db..000000000 --- a/mkdocs.yml +++ /dev/null @@ -1,49 +0,0 @@ -site_name: SD.Next Documentation -site_url: https://vladmandic.github.io/automatic -repo_url: https://github.com/vladmandic/automatic -repo_name: vladmandic/automatic -docs_dir: wiki - -theme: - name: material - - features: - - navigation.footer - - navigation.instant - - navigation.instant.prefetch - - navigation.instant.progress - - navigation.tracking - - palette: - - media: "(prefers-color-scheme: light)" - scheme: default - toggle: - icon: material/weather-night - name: Switch to dark mode - primary: teal - accent: pink - - - media: "(prefers-color-scheme: dark)" - scheme: slate - toggle: - icon: material/weather-sunny - name: Switch to light mode - primary: green - accent: yellow - - logo: assets/favicon.svg - favicon: assets/favicon.svg - -extra: - social: - - icon: fontawesome/brands/discord - link: https://discord.gg/VjvR2tabEX - name: Discord - - icon: fontawesome/brands/youtube - link: https://www.youtube.com/@SDNext - name: Youtube - -markdown_extensions: - - admonition - - pymdownx.details - - pymdownx.superfences \ No newline at end of file diff --git a/modules/extra_networks.py b/modules/extra_networks.py index fca48e21c..e96d2e5b7 100644 --- a/modules/extra_networks.py +++ b/modules/extra_networks.py @@ -74,9 +74,12 @@ def is_stepwise(en_obj): return any([len(str(x).split("@")) > 1 for x in all_args]) # noqa C419 # pylint: disable=use-a-generator -def activate(p, extra_network_data, step=0): +def activate(p, extra_network_data=None, step=0): """call activate for extra networks in extra_network_data in specified order, then call activate for all remaining registered networks with an empty argument list""" - if extra_network_data is None: + if p.disable_extra_networks: + return + extra_network_data = extra_network_data or p.network_data + if extra_network_data is None or len(extra_network_data) == 0: return stepwise = False for extra_network_args in extra_network_data.values(): @@ -106,15 +109,18 @@ def activate(p, extra_network_data, step=0): except Exception as e: errors.display(e, f"Activating network: type={extra_network_name}") - p.extra_network_data = extra_network_data + p.network_data = extra_network_data if stepwise: p.stepwise_lora = True shared.opts.data['lora_functional'] = functional -def deactivate(p, extra_network_data): +def deactivate(p, extra_network_data=None): """call deactivate for extra networks in extra_network_data in specified order, then call deactivate for all remaining registered networks""" - if extra_network_data is None: + if p.disable_extra_networks: + return + extra_network_data = extra_network_data or p.network_data + if extra_network_data is None or len(extra_network_data) == 0: return for extra_network_name in extra_network_data: extra_network = extra_network_registry.get(extra_network_name, None) diff --git a/modules/face/faceid.py b/modules/face/faceid.py index b74e15dc5..4a4f07531 100644 --- a/modules/face/faceid.py +++ b/modules/face/faceid.py @@ -204,7 +204,6 @@ def face_id( ip_model_dict["face_image"] = face_images ip_model_dict["faceid_embeds"] = face_embeds # overwrite placeholder faceid_model.set_scale(scale) - extra_network_data = None if p.all_prompts is None or len(p.all_prompts) == 0: processing.process_init(p) @@ -215,11 +214,9 @@ def face_id( p.negative_prompts = p.all_negative_prompts[n * p.batch_size:(n+1) * p.batch_size] p.seeds = p.all_seeds[n * p.batch_size:(n+1) * p.batch_size] p.subseeds = p.all_subseeds[n * p.batch_size:(n+1) * p.batch_size] - p.prompts, extra_network_data = extra_networks.parse_prompts(p.prompts) + p.prompts, p.network_data = extra_networks.parse_prompts(p.prompts) - if not p.disable_extra_networks: - with devices.autocast(): - extra_networks.activate(p, extra_network_data) + extra_networks.activate(p, p.network_data) ip_model_dict.update({ "prompt": p.prompts[0], "negative_prompt": p.negative_prompts[0], @@ -239,8 +236,7 @@ def face_id( devices.torch_gc() ipadapter.unapply(p.sd_model) - if not p.disable_extra_networks: - extra_networks.deactivate(p, extra_network_data) + extra_networks.deactivate(p, p.network_data) p.extra_generation_params["IP Adapter"] = f"{basename}:{scale}" finally: diff --git a/modules/lora/extra_networks_lora.py b/modules/lora/extra_networks_lora.py index 57966550a..4ce7a94a9 100644 --- a/modules/lora/extra_networks_lora.py +++ b/modules/lora/extra_networks_lora.py @@ -1,5 +1,4 @@ import re -import time import numpy as np import modules.lora.networks as networks from modules import extra_networks, shared @@ -128,10 +127,9 @@ def activate(self, p, params_list, step=0): if len(networks.loaded_networks) > 0 and step == 0: infotext(p) prompt(p) - shared.log.info(f'Load network: type=LoRA apply={[n.name for n in networks.loaded_networks]} te={te_multipliers} unet={unet_multipliers} time={networks.get_timers()}') + shared.log.info(f'Load network: type=LoRA apply={[n.name for n in networks.loaded_networks]} te={te_multipliers} unet={unet_multipliers} time={networks.timer.summary}') def deactivate(self, p): - t0 = time.time() if shared.native and len(networks.diffuser_loaded) > 0: if hasattr(shared.sd_model, "unload_lora_weights") and hasattr(shared.sd_model, "text_encoder"): if not (shared.compiled_model_state is not None and shared.compiled_model_state.is_compiled is True): @@ -142,10 +140,8 @@ def deactivate(self, p): except Exception: pass networks.network_deactivate() - t1 = time.time() - networks.timer['restore'] += t1 - t0 if self.active and networks.debug: - shared.log.debug(f"Network end: type=LoRA load={networks.timer['load']:.2f} apply={networks.timer['apply']:.2f} restore={networks.timer['restore']:.2f}") + shared.log.debug(f"Network end: type=LoRA time={networks.timer.summary}") if self.errors: for k, v in self.errors.items(): shared.log.error(f'LoRA: name="{k}" errors={v}') diff --git a/modules/lora/lora_timers.py b/modules/lora/lora_timers.py new file mode 100644 index 000000000..30c35a728 --- /dev/null +++ b/modules/lora/lora_timers.py @@ -0,0 +1,38 @@ +class Timer(): + list: float = 0 + load: float = 0 + backup: float = 0 + calc: float = 0 + apply: float = 0 + move: float = 0 + restore: float = 0 + activate: float = 0 + deactivate: float = 0 + + @property + def total(self): + return round(self.activate + self.deactivate, 2) + + @property + def summary(self): + t = {} + for k, v in self.__dict__.items(): + if v > 0.1: + t[k] = round(v, 2) + return t + + def clear(self, complete: bool = False): + self.backup = 0 + self.calc = 0 + self.apply = 0 + self.move = 0 + self.restore = 0 + if complete: + self.activate = 0 + self.deactivate = 0 + + def add(self, name, t): + self.__dict__[name] += t + + def __str__(self): + return f'{self.__class__.__name__}({self.summary})' diff --git a/modules/lora/network_lora.py b/modules/lora/network_lora.py index 6c1d7ea3f..8bf475ebc 100644 --- a/modules/lora/network_lora.py +++ b/modules/lora/network_lora.py @@ -22,7 +22,6 @@ def __init__(self, net: network.Network, weights: network.NetworkWeights): self.dim = weights.w["lora_down.weight"].shape[0] def create_module(self, weights, key, none_ok=False): - from modules.shared import opts weight = weights.get(key) if weight is None and none_ok: return None @@ -49,8 +48,6 @@ def create_module(self, weights, key, none_ok=False): if weight.shape != module.weight.shape: weight = weight.reshape(module.weight.shape) module.weight.copy_(weight) - if opts.lora_load_gpu: - module = module.to(device=devices.device, dtype=devices.dtype) module.weight.requires_grad_(False) return module diff --git a/modules/lora/networks.py b/modules/lora/networks.py index 9618f01a9..805b24b52 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -1,24 +1,15 @@ from typing import Union, List +from contextlib import nullcontext import os import re import time import concurrent -from contextlib import nullcontext import torch import diffusers.models.lora import rich.progress as rp -import modules.lora.network as network -import modules.lora.network_lora as network_lora -import modules.lora.network_hada as network_hada -import modules.lora.network_ia3 as network_ia3 -import modules.lora.network_oft as network_oft -import modules.lora.network_lokr as network_lokr -import modules.lora.network_full as network_full -import modules.lora.network_norm as network_norm -import modules.lora.network_glora as network_glora -import modules.lora.network_overrides as network_overrides -import modules.lora.lora_convert as lora_convert +from modules.lora import lora_timers, network, lora_convert, network_overrides +from modules.lora import network_lora, network_hada, network_ia3, network_oft, network_lokr, network_full, network_norm, network_glora from modules.lora.extra_networks_lora import ExtraNetworkLora from modules import shared, devices, sd_models, sd_models_compile, errors, files_cache, model_quant @@ -28,7 +19,6 @@ available_networks = {} available_network_aliases = {} loaded_networks: List[network.Network] = [] -timer = { 'list': 0, 'load': 0, 'backup': 0, 'calc': 0, 'apply': 0, 'move': 0, 'restore': 0, 'deactivate': 0 } bnb = None lora_cache = {} diffuser_loaded = [] @@ -36,6 +26,7 @@ available_network_hash_lookup = {} forbidden_network_aliases = {} re_network_name = re.compile(r"(.*)\s*\([0-9a-fA-F]+\)") +timer = lora_timers.Timer() module_types = [ network_lora.ModuleTypeLora(), network_hada.ModuleTypeHada(), @@ -47,19 +38,6 @@ network_glora.ModuleTypeGLora(), ] - -def total_time(): - return sum(timer.values()) - - -def get_timers(): - t = { 'total': round(sum(timer.values()), 2) } - for k, v in timer.items(): - if v > 0.1: - t[k] = round(v, 2) - return t - - # section: load networks from disk def load_diffusers(name, network_on_disk, lora_scale=shared.opts.extra_networks_default_multiplier) -> Union[network.Network, None]: @@ -154,7 +132,7 @@ def load_safetensors(name, network_on_disk) -> Union[network.Network, None]: if debug: shared.log.debug(f'LoRA name="{name}" unmatched={keys_failed_to_match}') else: - shared.log.debug(f'LoRA name="{name}" type={set(network_types)} keys={len(matched_networks)}') + shared.log.debug(f'LoRA name="{name}" type={set(network_types)} keys={len(matched_networks)} direct={shared.opts.lora_fuse_diffusers}') if len(matched_networks) == 0: return None lora_cache[name] = net @@ -222,12 +200,11 @@ def add_network(filename): for fn in candidates: executor.submit(add_network, fn) t1 = time.time() - timer['list'] = t1 - t0 + timer.list = t1 - t0 shared.log.info(f'Available LoRAs: path="{shared.cmd_opts.lora_dir}" items={len(available_networks)} folders={len(forbidden_network_aliases)} time={t1 - t0:.2f}') def network_load(names, te_multipliers=None, unet_multipliers=None, dyn_dims=None): - timer['list'] = 0 networks_on_disk: list[network.NetworkOnDisk] = [available_network_aliases.get(name, None) for name in names] if any(x is None for x in networks_on_disk): list_available_networks() @@ -301,13 +278,12 @@ def network_load(names, te_multipliers=None, unet_multipliers=None, dyn_dims=Non if len(loaded_networks) > 0: devices.torch_gc() - t1 = time.time() - timer['load'] = t1 - t0 + timer.load = time.time() - t0 # section: process loaded networks -def network_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv], weight: torch.Tensor, network_layer_name: str, wanted_names: tuple): +def network_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv], network_layer_name: str, wanted_names: tuple): global bnb # pylint: disable=W0603 backup_size = 0 if len(loaded_networks) > 0 and network_layer_name is not None and any([net.modules.get(network_layer_name, None) for net in loaded_networks]): # noqa: C419 # pylint: disable=R1729 @@ -315,9 +291,10 @@ def network_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.n weights_backup = getattr(self, "network_weights_backup", None) if weights_backup is None and wanted_names != (): # pylint: disable=C1803 + weight = getattr(self, 'weight', None) self.network_weights_backup = None - if shared.opts.lora_fuse_diffusers or shared.opts.lora_low_memory: - weights_backup = True + if shared.opts.lora_fuse_diffusers: + self.network_weights_backup = True elif getattr(weight, "quant_type", None) in ['nf4', 'fp4']: if bnb is None: bnb = model_quant.load_bnb('Load network: type=LoRA', silent=True) @@ -329,86 +306,112 @@ def network_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.n self.blocksize = weight.blocksize else: weights_backup = weight.clone() + weights_backup = weights_backup.to(devices.cpu) else: weights_backup = weight.clone() - if shared.opts.lora_offload_backup and weights_backup is not None and isinstance(weights_backup, torch.Tensor): weights_backup = weights_backup.to(devices.cpu) - self.network_weights_backup = weights_backup bias_backup = getattr(self, "network_bias_backup", None) if bias_backup is None: if getattr(self, 'bias', None) is not None: - if shared.opts.lora_fuse_diffusers or shared.opts.lora_low_memory: - bias_backup = True + if shared.opts.lora_fuse_diffusers: + self.network_bias_backup = True else: bias_backup = self.bias.clone() - else: - bias_backup = None - if shared.opts.lora_offload_backup and bias_backup is not None and isinstance(bias_backup, torch.Tensor): - bias_backup = bias_backup.to(devices.cpu) - self.network_bias_backup = bias_backup + bias_backup = bias_backup.to(devices.cpu) if getattr(self, 'network_weights_backup', None) is not None: backup_size += self.network_weights_backup.numel() * self.network_weights_backup.element_size() if isinstance(self.network_weights_backup, torch.Tensor) else 0 if getattr(self, 'network_bias_backup', None) is not None: backup_size += self.network_bias_backup.numel() * self.network_bias_backup.element_size() if isinstance(self.network_bias_backup, torch.Tensor) else 0 - t1 = time.time() - timer['backup'] += t1 - t0 + timer.backup += time.time() - t0 return backup_size -def network_calc_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv], weight: torch.Tensor, network_layer_name: str): +def network_calc_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv], network_layer_name: str): if shared.opts.diffusers_offload_mode == "none": self.to(devices.device) batch_updown = None batch_ex_bias = None for net in loaded_networks: module = net.modules.get(network_layer_name, None) - if module is not None and hasattr(self, 'weight'): - try: + if module is None: + continue + try: + t0 = time.time() + weight = self.weight.to(devices.device) + updown, ex_bias = module.calc_updown(weight) + if batch_updown is not None and updown is not None: + batch_updown += updown.to(batch_updown.device) + else: + batch_updown = updown + if batch_ex_bias is not None and ex_bias is not None: + batch_ex_bias += ex_bias.to(batch_ex_bias.device) + else: + batch_ex_bias = ex_bias + timer.calc += time.time() - t0 + if shared.opts.diffusers_offload_mode == "sequential": t0 = time.time() - updown, ex_bias = module.calc_updown(weight) + if batch_updown is not None: + batch_updown = batch_updown.to(devices.cpu) + if batch_ex_bias is not None: + batch_ex_bias = batch_ex_bias.to(devices.cpu) t1 = time.time() - if batch_updown is not None and updown is not None: - batch_updown += updown.to(batch_updown.device) - else: - batch_updown = updown - if batch_ex_bias is not None and ex_bias is not None: - batch_ex_bias += ex_bias.to(batch_ex_bias.device) - else: - batch_ex_bias = ex_bias - timer['calc'] += t1 - t0 - if shared.opts.diffusers_offload_mode != "none": - t0 = time.time() - if batch_updown is not None: - batch_updown = batch_updown.to(devices.cpu) - if batch_ex_bias is not None: - batch_ex_bias = batch_ex_bias.to(devices.cpu) - t1 = time.time() - timer['move'] += t1 - t0 - except RuntimeError as e: - extra_network_lora.errors[net.name] = extra_network_lora.errors.get(net.name, 0) + 1 - if debug: - module_name = net.modules.get(network_layer_name, None) - shared.log.error(f'LoRA apply weight name="{net.name}" module="{module_name}" layer="{network_layer_name}" {e}') - errors.display(e, 'LoRA') - raise RuntimeError('LoRA apply weight') from e - continue + timer.move += t1 - t0 + except RuntimeError as e: + extra_network_lora.errors[net.name] = extra_network_lora.errors.get(net.name, 0) + 1 + if debug: + module_name = net.modules.get(network_layer_name, None) + shared.log.error(f'LoRA apply weight name="{net.name}" module="{module_name}" layer="{network_layer_name}" {e}') + errors.display(e, 'LoRA') + raise RuntimeError('LoRA apply weight') from e + continue return batch_updown, batch_ex_bias -def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv], updown: torch.Tensor, ex_bias: torch.Tensor, orig_device: torch.device, deactivate: bool = False): +def network_apply_direct(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv], updown: torch.Tensor, ex_bias: torch.Tensor, deactivate: bool = False): + weights_backup = getattr(self, "network_weights_backup", False) + bias_backup = getattr(self, "network_bias_backup", False) + if not weights_backup and not bias_backup: + return None, None t0 = time.time() + + if weights_backup: + if updown is not None and len(self.weight.shape) == 4 and self.weight.shape[1] == 9: # inpainting model. zero pad updown to make channel[1] 4 to 9 + updown = torch.nn.functional.pad(updown, (0, 0, 0, 0, 0, 5)) # pylint: disable=not-callable + if updown is not None: + if deactivate: + updown *= -1 + new_weight = self.weight.to(devices.device) + updown.to(devices.device) + if getattr(self, "quant_type", None) in ['nf4', 'fp4'] and bnb is not None: + self.weight = bnb.nn.Params4bit(new_weight, quant_state=self.quant_state, quant_type=self.quant_type, blocksize=self.blocksize) + else: + self.weight = torch.nn.Parameter(new_weight, requires_grad=False) + del new_weight + if hasattr(self, "qweight") and hasattr(self, "freeze"): + self.freeze() + + if bias_backup: + if ex_bias is not None: + if deactivate: + ex_bias *= -1 + new_weight = bias_backup.to(devices.device) + ex_bias.to(devices.device) + self.bias = torch.nn.Parameter(new_weight, requires_grad=False) + del new_weight + + timer.apply += time.time() - t0 + return self.weight.device, self.weight.dtype + + +def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv], updown: torch.Tensor, ex_bias: torch.Tensor, orig_device: torch.device, deactivate: bool = False): weights_backup = getattr(self, "network_weights_backup", None) bias_backup = getattr(self, "network_bias_backup", None) if weights_backup is None and bias_backup is None: return None, None + t0 = time.time() if weights_backup is not None: - if isinstance(weights_backup, bool): - weights_backup = self.weight - else: - self.weight = None + self.weight = None if updown is not None and len(weights_backup.shape) == 4 and weights_backup.shape[1] == 9: # inpainting model. zero pad updown to make channel[1] 4 to 9 updown = torch.nn.functional.pad(updown, (0, 0, 0, 0, 0, 5)) # pylint: disable=not-callable if updown is not None: @@ -426,10 +429,7 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn self.freeze() if bias_backup is not None: - if isinstance(bias_backup, bool): - bias_backup = self.bias - else: - self.bias = None + self.bias = None if ex_bias is not None: if deactivate: ex_bias *= -1 @@ -438,19 +438,16 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn del new_weight else: self.bias = torch.nn.Parameter(bias_backup.to(device=orig_device), requires_grad=False) - else: - self.bias = None - t1 = time.time() - timer['apply'] += t1 - t0 + timer.apply += time.time() - t0 return self.weight.device, self.weight.dtype def network_deactivate(): - if not shared.opts.lora_low_memory: + if not shared.opts.lora_fuse_diffusers: return - timer['deactivate'] = 0 t0 = time.time() + timer.clear() sd_model = getattr(shared.sd_model, "pipe", shared.sd_model) # wrapped model compatiblility if shared.opts.diffusers_offload_mode == "sequential": sd_models.disable_offload(sd_model) @@ -462,15 +459,13 @@ def network_deactivate(): modules[component_name] = list(component.named_modules()) total = sum(len(x) for x in modules.values()) if len(loaded_networks) > 0: - pbar = rp.Progress(rp.TextColumn('[cyan]Deactivate network: type=LoRA'), rp.BarColumn(), rp.TaskProgressColumn(), - rp.TimeRemainingColumn(), rp.TimeElapsedColumn(), rp.TextColumn('[cyan]{task.description}'), - console=shared.console) + pbar = rp.Progress(rp.TextColumn('[cyan]Network: type=LoRA action=deactivate'), rp.BarColumn(), rp.TaskProgressColumn(), rp.TimeRemainingColumn(), rp.TimeElapsedColumn(), rp.TextColumn('[cyan]{task.description}'), console=shared.console) task = pbar.add_task(description='', total=total) else: task = None pbar = nullcontext() with devices.inference_context(), pbar: - applied = 0 + applied_layers = [] weights_devices = [] weights_dtypes = [] for component in modules.keys(): @@ -479,36 +474,33 @@ def network_deactivate(): network_layer_name = getattr(module, 'network_layer_name', None) if shared.state.interrupted or network_layer_name is None: if task is not None: - pbar.update(task, advance=1, description=f'networks={len(loaded_networks)} skip') + pbar.update(task, advance=1) continue - weight = getattr(module, 'weight', None) - weight = weight.to(devices.device) if weight is not None else None - batch_updown, batch_ex_bias = network_calc_weights(module, weight, network_layer_name) - weights_device, weights_dtype = network_apply_weights(module, batch_updown, batch_ex_bias, orig_device, deactivate=True) + batch_updown, batch_ex_bias = network_calc_weights(module, network_layer_name) + if shared.opts.lora_fuse_diffusers: + weights_device, weights_dtype = network_apply_direct(module, batch_updown, batch_ex_bias, deactivate=True) + else: + weights_device, weights_dtype = network_apply_weights(module, batch_updown, batch_ex_bias, orig_device, deactivate=True) weights_devices.append(weights_device) weights_dtypes.append(weights_dtype) if batch_updown is not None or batch_ex_bias is not None: - applied += 1 - del weight, batch_updown, batch_ex_bias + applied_layers.append(network_layer_name) + del batch_updown, batch_ex_bias module.network_current_names = () if task is not None: - pbar.update(task, advance=1, - description=f'networks={len(loaded_networks)} modules={len(modules)} deactivate={applied}') + pbar.update(task, advance=1, description=f'networks={len(loaded_networks)} modules={len(modules)} deactivate={len(applied_layers)}') weights_devices, weights_dtypes = list(set([x for x in weights_devices if x is not None])), list(set([x for x in weights_dtypes if x is not None])) # noqa: C403 # pylint: disable=R1718 + timer.deactivate = time.time() - t0 if debug and len(loaded_networks) > 0: - shared.log.debug( - f'Deactivate network: type=LoRA networks={len(loaded_networks)} modules={total} deactivate={applied} device={weights_devices} dtype={weights_dtypes} fuse={shared.opts.lora_fuse_diffusers} time={get_timers()}') + shared.log.debug(f'Deactivate network: type=LoRA networks={len(loaded_networks)} modules={total} deactivate={len(applied_layers)} device={weights_devices} dtype={weights_dtypes} fuse={shared.opts.lora_fuse_diffusers} time={timer.summary}') modules.clear() if shared.opts.diffusers_offload_mode == "sequential": sd_models.set_diffuser_offload(sd_model, op="model") - t1 = time.time() - timer['deactivate'] += t1 - t0 + def network_activate(): - timer['backup'] = 0 - timer['calc'] = 0 - timer['apply'] = 0 - timer['move'] = 0 + t0 = time.time() + timer.clear(complete=True) sd_model = getattr(shared.sd_model, "pipe", shared.sd_model) # wrapped model compatiblility if shared.opts.diffusers_offload_mode == "sequential": sd_models.disable_offload(sd_model) @@ -520,14 +512,14 @@ def network_activate(): modules[component_name] = list(component.named_modules()) total = sum(len(x) for x in modules.values()) if len(loaded_networks) > 0: - pbar = rp.Progress(rp.TextColumn('[cyan]Apply network: type=LoRA'), rp.BarColumn(), rp.TaskProgressColumn(), rp.TimeRemainingColumn(), rp.TimeElapsedColumn(), rp.TextColumn('[cyan]{task.description}'), console=shared.console) + pbar = rp.Progress(rp.TextColumn('[cyan]Network: type=LoRA action=activate'), rp.BarColumn(), rp.TaskProgressColumn(), rp.TimeRemainingColumn(), rp.TimeElapsedColumn(), rp.TextColumn('[cyan]{task.description}'), console=shared.console) task = pbar.add_task(description='' , total=total) else: task = None pbar = nullcontext() with devices.inference_context(), pbar: wanted_names = tuple((x.name, x.te_multiplier, x.unet_multiplier, x.dyn_dim) for x in loaded_networks) if len(loaded_networks) > 0 else () - applied = 0 + applied_layers = [] backup_size = 0 weights_devices = [] weights_dtypes = [] @@ -536,26 +528,28 @@ def network_activate(): for _, module in modules[component]: network_layer_name = getattr(module, 'network_layer_name', None) current_names = getattr(module, "network_current_names", ()) - if shared.state.interrupted or network_layer_name is None or current_names == wanted_names: + if getattr(module, 'weight', None) is None or shared.state.interrupted or network_layer_name is None or current_names == wanted_names: if task is not None: - pbar.update(task, advance=1, description=f'networks={len(loaded_networks)} skip') + pbar.update(task, advance=1) continue - weight = getattr(module, 'weight', None) - weight = weight.to(devices.device) if weight is not None else None - backup_size += network_backup_weights(module, weight, network_layer_name, wanted_names) - batch_updown, batch_ex_bias = network_calc_weights(module, weight, network_layer_name) - weights_device, weights_dtype = network_apply_weights(module, batch_updown, batch_ex_bias, orig_device) + backup_size += network_backup_weights(module, network_layer_name, wanted_names) + batch_updown, batch_ex_bias = network_calc_weights(module, network_layer_name) + if shared.opts.lora_fuse_diffusers: + weights_device, weights_dtype = network_apply_direct(module, batch_updown, batch_ex_bias) + else: + weights_device, weights_dtype = network_apply_weights(module, batch_updown, batch_ex_bias, orig_device) weights_devices.append(weights_device) weights_dtypes.append(weights_dtype) if batch_updown is not None or batch_ex_bias is not None: - applied += 1 - del weight, batch_updown, batch_ex_bias + applied_layers.append(network_layer_name) + del batch_updown, batch_ex_bias module.network_current_names = wanted_names if task is not None: - pbar.update(task, advance=1, description=f'networks={len(loaded_networks)} modules={len(modules)} apply={applied} backup={backup_size}') + pbar.update(task, advance=1, description=f'networks={len(loaded_networks)} modules={total} apply={len(applied_layers)} backup={backup_size}') weights_devices, weights_dtypes = list(set([x for x in weights_devices if x is not None])), list(set([x for x in weights_dtypes if x is not None])) # noqa: C403 # pylint: disable=R1718 + timer.activate = time.time() - t0 if debug and len(loaded_networks) > 0: - shared.log.debug(f'Load network: type=LoRA networks={len(loaded_networks)} modules={total} apply={applied} device={weights_devices} dtype={weights_dtypes} backup={backup_size} fuse={shared.opts.lora_fuse_diffusers} time={get_timers()}') + shared.log.debug(f'Load network: type=LoRA networks={len(loaded_networks)} modules={total} apply={len(applied_layers)} device={weights_devices} dtype={weights_dtypes} backup={backup_size} fuse={shared.opts.lora_fuse_diffusers} time={timer.summary}') modules.clear() if shared.opts.diffusers_offload_mode == "sequential": sd_models.set_diffuser_offload(sd_model, op="model") diff --git a/modules/processing.py b/modules/processing.py index 57512850a..7ae397538 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -286,7 +286,6 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: t0 = time.time() if not hasattr(p, 'skip_init'): p.init(p.all_prompts, p.all_seeds, p.all_subseeds) - extra_network_data = None debug(f'Processing inner: args={vars(p)}') for n in range(p.n_iter): pag.apply(p) @@ -311,9 +310,9 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: p.scripts.before_process_batch(p, batch_number=n, prompts=p.prompts, seeds=p.seeds, subseeds=p.subseeds) if len(p.prompts) == 0: break - p.prompts, extra_network_data = extra_networks.parse_prompts(p.prompts) - if not p.disable_extra_networks: - extra_networks.activate(p, extra_network_data) + p.prompts, p.network_data = extra_networks.parse_prompts(p.prompts) + if not shared.native: + extra_networks.activate(p, p.network_data) if p.scripts is not None and isinstance(p.scripts, scripts.ScriptRunner): p.scripts.process_batch(p, batch_number=n, prompts=p.prompts, seeds=p.seeds, subseeds=p.subseeds) @@ -417,6 +416,10 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: timer.process.record('post') del samples + + if not shared.native: + extra_networks.deactivate(p, p.network_data) + devices.torch_gc() if hasattr(shared.sd_model, 'restore_pipeline') and shared.sd_model.restore_pipeline is not None: @@ -445,9 +448,6 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: from modules import ipadapter ipadapter.unapply(shared.sd_model) - if not p.disable_extra_networks: - extra_networks.deactivate(p, extra_network_data) - if shared.opts.include_mask: if shared.opts.mask_apply_overlay and p.overlay_images is not None and len(p.overlay_images): p.image_mask = create_binary_mask(p.overlay_images[0]) diff --git a/modules/processing_args.py b/modules/processing_args.py index d73762d29..93b0bf9b2 100644 --- a/modules/processing_args.py +++ b/modules/processing_args.py @@ -101,6 +101,7 @@ def task_specific_kwargs(p, model): def set_pipeline_args(p, model, prompts: list, negative_prompts: list, prompts_2: typing.Optional[list]=None, negative_prompts_2: typing.Optional[list]=None, desc:str='', **kwargs): t0 = time.time() + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) apply_circular(p.tiling, model) if hasattr(model, "set_progress_bar_config"): model.set_progress_bar_config(bar_format='Progress {rate_fmt}{postfix} {bar} {percentage:3.0f}% {n_fmt}/{total_fmt} {elapsed} {remaining} ' + '\x1b[38;5;71m' + desc, ncols=80, colour='#327fba') diff --git a/modules/processing_callbacks.py b/modules/processing_callbacks.py index f3eb0bc37..0b4c7dfe1 100644 --- a/modules/processing_callbacks.py +++ b/modules/processing_callbacks.py @@ -67,7 +67,7 @@ def diffusers_callback(pipe, step: int = 0, timestep: int = 0, kwargs: dict = {} raise AssertionError('Interrupted...') time.sleep(0.1) if hasattr(p, "stepwise_lora") and shared.native: - extra_networks.activate(p, p.extra_network_data, step=step) + extra_networks.activate(p, step=step) if latents is None: return kwargs elif shared.opts.nan_skip: diff --git a/modules/processing_class.py b/modules/processing_class.py index 21e86c1b0..2cbc07cc2 100644 --- a/modules/processing_class.py +++ b/modules/processing_class.py @@ -139,6 +139,7 @@ def __init__(self, self.negative_pooleds = [] self.disable_extra_networks = False self.iteration = 0 + self.network_data = {} # initializers self.prompt = prompt diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 0341cac4d..d22a9de97 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -4,7 +4,7 @@ import numpy as np import torch import torchvision.transforms.functional as TF -from modules import shared, devices, processing, sd_models, errors, sd_hijack_hypertile, processing_vae, sd_models_compile, hidiffusion, timer, modelstats +from modules import shared, devices, processing, sd_models, errors, sd_hijack_hypertile, processing_vae, sd_models_compile, hidiffusion, timer, modelstats, extra_networks from modules.processing_helpers import resize_hires, calculate_base_steps, calculate_hires_steps, calculate_refiner_steps, save_intermediate, update_sampler, is_txt2img, is_refiner_enabled from modules.processing_args import set_pipeline_args from modules.onnx_impl import preprocess_pipeline as preprocess_onnx_pipeline, check_parameters_changed as olive_check_parameters_changed @@ -89,6 +89,7 @@ def process_base(p: processing.StableDiffusionProcessing): sd_models.move_model(shared.sd_model.unet, devices.device) if hasattr(shared.sd_model, 'transformer'): sd_models.move_model(shared.sd_model.transformer, devices.device) + extra_networks.activate(p) hidiffusion.apply(p, shared.sd_model_type) # if 'image' in base_args: # base_args['image'] = set_latents(p) @@ -223,11 +224,14 @@ def process_hires(p: processing.StableDiffusionProcessing, output): shared.state.job = 'HiRes' shared.state.sampling_steps = hires_args.get('prior_num_inference_steps', None) or p.steps or hires_args.get('num_inference_steps', None) try: + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) sd_models.move_model(shared.sd_model, devices.device) if hasattr(shared.sd_model, 'unet'): sd_models.move_model(shared.sd_model.unet, devices.device) if hasattr(shared.sd_model, 'transformer'): sd_models.move_model(shared.sd_model.transformer, devices.device) + if 'base' in p.skip: + extra_networks.activate(p) sd_models_compile.check_deepcache(enable=True) output = shared.sd_model(**hires_args) # pylint: disable=not-callable if isinstance(output, dict): @@ -345,6 +349,7 @@ def process_refine(p: processing.StableDiffusionProcessing, output): def process_decode(p: processing.StableDiffusionProcessing, output): + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) if output is not None: if not hasattr(output, 'images') and hasattr(output, 'frames'): shared.log.debug(f'Generated: frames={len(output.frames[0])}') @@ -405,8 +410,6 @@ def process_diffusers(p: processing.StableDiffusionProcessing): shared.sd_model = orig_pipeline return results - shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) - # sanitize init_images if hasattr(p, 'init_images') and getattr(p, 'init_images', None) is None: del p.init_images @@ -453,13 +456,13 @@ def process_diffusers(p: processing.StableDiffusionProcessing): shared.sd_model = orig_pipeline return results - results = process_decode(p, output) + extra_networks.deactivate(p) + timer.process.add('lora', networks.timer.total) + results = process_decode(p, output) timer.process.record('decode') - timer.process.add('lora', networks.total_time()) shared.sd_model = orig_pipeline - shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) if p.state == '': diff --git a/modules/prompt_parser_diffusers.py b/modules/prompt_parser_diffusers.py index 06c0b6012..8c140e0d6 100644 --- a/modules/prompt_parser_diffusers.py +++ b/modules/prompt_parser_diffusers.py @@ -92,7 +92,7 @@ def flatten(xss): return [x for xs in xss for x in xs] # unpack EN data in case of TE LoRA - en_data = p.extra_network_data + en_data = p.network_data en_data = [idx.items for item in en_data.values() for idx in item] effective_batch = 1 if self.allsame else self.batchsize key = str([self.prompts, self.negative_prompts, effective_batch, self.clip_skip, self.steps, en_data]) diff --git a/modules/shared.py b/modules/shared.py index 83c1cc0b3..92c0388a4 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -905,15 +905,12 @@ def get_default_modes(): "extra_networks_default_multiplier": OptionInfo(1.0, "Default strength", gr.Slider, {"minimum": 0.0, "maximum": 2.0, "step": 0.01}), "lora_preferred_name": OptionInfo("filename", "LoRA preferred name", gr.Radio, {"choices": ["filename", "alias"], "visible": False}), "lora_add_hashes_to_infotext": OptionInfo(False, "LoRA add hash info"), - "lora_fuse_diffusers": OptionInfo(False if not cmd_opts.use_openvino else True, "LoRA fuse directly to model"), - "lora_load_gpu": OptionInfo(True if not (cmd_opts.lowvram or cmd_opts.medvram) else False, "LoRA load directly to GPU"), - "lora_offload_backup": OptionInfo(True, "LoRA offload backup weights"), + "lora_fuse_diffusers": OptionInfo(True, "LoRA fuse directly to model"), "lora_force_diffusers": OptionInfo(False if not cmd_opts.use_openvino else True, "LoRA force loading of all models using Diffusers"), "lora_maybe_diffusers": OptionInfo(False, "LoRA force loading of specific models using Diffusers"), "lora_apply_tags": OptionInfo(0, "LoRA auto-apply tags", gr.Slider, {"minimum": -1, "maximum": 32, "step": 1}), "lora_in_memory_limit": OptionInfo(0, "LoRA memory cache", gr.Slider, {"minimum": 0, "maximum": 24, "step": 1}), "lora_quant": OptionInfo("NF4","LoRA precision in quantized models", gr.Radio, {"choices": ["NF4", "FP4"]}), - "lora_low_memory": OptionInfo(False, "LoRA low memory mode"), })) options_templates.update(options_section((None, "Internal options"), { diff --git a/wiki b/wiki index c5d484397..20c9fe52f 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit c5d484397f7504fdea098d5e24c843a69c9fd2a2 +Subproject commit 20c9fe52f253c23e736227787ddebd4cbfcbfe68 From 8ac5834b1cb9fc417d85c5e78846e1415fcadd74 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 9 Dec 2024 13:46:25 -0500 Subject: [PATCH 071/249] update hotkeys Signed-off-by: Vladimir Mandic --- javascript/script.js | 5 +++-- wiki | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/javascript/script.js b/javascript/script.js index 250e90ba2..836d9b102 100644 --- a/javascript/script.js +++ b/javascript/script.js @@ -125,11 +125,12 @@ document.addEventListener('keydown', (e) => { let elem; if (e.key === 'Escape') elem = getUICurrentTabContent().querySelector('button[id$=_interrupt]'); if (e.key === 'Enter' && e.ctrlKey) elem = getUICurrentTabContent().querySelector('button[id$=_generate]'); - if (e.key === 'Backspace' && e.ctrlKey) elem = getUICurrentTabContent().querySelector('button[id$=_reprocess]'); + if (e.key === 'r' && e.ctrlKey) elem = getUICurrentTabContent().querySelector('button[id$=_reprocess]'); if (e.key === ' ' && e.ctrlKey) elem = getUICurrentTabContent().querySelector('button[id$=_extra_networks_btn]'); + if (e.key === 'n' && e.ctrlKey) elem = getUICurrentTabContent().querySelector('button[id$=_extra_networks_btn]'); if (e.key === 's' && e.ctrlKey) elem = getUICurrentTabContent().querySelector('button[id^=save_]'); if (e.key === 'Insert' && e.ctrlKey) elem = getUICurrentTabContent().querySelector('button[id^=save_]'); - if (e.key === 'Delete' && e.ctrlKey) elem = getUICurrentTabContent().querySelector('button[id^=delete_]'); + if (e.key === 'd' && e.ctrlKey) elem = getUICurrentTabContent().querySelector('button[id^=delete_]'); // if (e.key === 'm' && e.ctrlKey) elem = gradioApp().getElementById('setting_sd_model_checkpoint'); if (elem) { e.preventDefault(); diff --git a/wiki b/wiki index 20c9fe52f..8960da514 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 20c9fe52f253c23e736227787ddebd4cbfcbfe68 +Subproject commit 8960da514e9aff4a5d47402925c9498536443379 From 6c87458c537f91d999e7194da544b9ce69fd3ab1 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 9 Dec 2024 14:16:26 -0500 Subject: [PATCH 072/249] fix preview choice Signed-off-by: Vladimir Mandic --- modules/sd_samplers_common.py | 6 ++---- modules/shared_state.py | 1 - 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/modules/sd_samplers_common.py b/modules/sd_samplers_common.py index f6f6c18d5..a96795a25 100644 --- a/modules/sd_samplers_common.py +++ b/modules/sd_samplers_common.py @@ -35,7 +35,6 @@ def setup_img2img_steps(p, steps=None): def single_sample_to_image(sample, approximation=None): with queue_lock: t0 = time.time() - sd_cascade = False if approximation is None: approximation = approximation_indexes.get(shared.opts.show_progress_type, None) if approximation is None: @@ -50,10 +49,9 @@ def single_sample_to_image(sample, approximation=None): if len(sample.shape) > 4: # likely unknown video latent (e.g. svd) return Image.new(mode="RGB", size=(512, 512)) - if len(sample) == 16: # sd_cascade - sd_cascade = True if len(sample.shape) == 4 and sample.shape[0]: # likely animatediff latent sample = sample.permute(1, 0, 2, 3)[0] + # TODO remove if shared.native: # [-x,x] to [-5,5] sample_max = torch.max(sample) if sample_max > 5: @@ -65,7 +63,7 @@ def single_sample_to_image(sample, approximation=None): if approximation == 2: # TAESD x_sample = sd_vae_taesd.decode(sample) x_sample = (1.0 + x_sample) / 2.0 # preview requires smaller range - elif sd_cascade and approximation != 3: + elif shared.sd_model_type == 'sc' and approximation != 3: x_sample = sd_vae_stablecascade.decode(sample) elif approximation == 0: # Simple x_sample = sd_vae_approx.cheap_approximation(sample) * 0.5 + 0.5 diff --git a/modules/shared_state.py b/modules/shared_state.py index 51d33f9ed..3d3cb1ae6 100644 --- a/modules/shared_state.py +++ b/modules/shared_state.py @@ -141,7 +141,6 @@ def set_current_image(self): if self.job == 'VAE': # avoid generating preview while vae is running return from modules.shared import opts, cmd_opts - """sets self.current_image from self.current_latent if enough sampling steps have been made after the last call to this""" if cmd_opts.lowvram or self.api: return if abs(self.sampling_step - self.current_image_sampling_step) >= opts.show_progress_every_n_steps and opts.live_previews_enable and opts.show_progress_every_n_steps > 0: From d40dfc08fccbbc19df5eaa07cd6ca42c096bdfea Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 9 Dec 2024 15:23:22 -0500 Subject: [PATCH 073/249] lora split te apply Signed-off-by: Vladimir Mandic --- modules/extra_networks.py | 45 ++++++++++++++++------------- modules/lora/extra_networks_lora.py | 6 ++-- modules/lora/networks.py | 19 +++++++----- modules/processing_args.py | 3 +- modules/processing_diffusers.py | 2 +- 5 files changed, 43 insertions(+), 32 deletions(-) diff --git a/modules/extra_networks.py b/modules/extra_networks.py index e96d2e5b7..fe141cca1 100644 --- a/modules/extra_networks.py +++ b/modules/extra_networks.py @@ -1,6 +1,7 @@ import re +import inspect from collections import defaultdict -from modules import errors, shared, devices +from modules import errors, shared extra_network_registry = {} @@ -74,7 +75,7 @@ def is_stepwise(en_obj): return any([len(str(x).split("@")) > 1 for x in all_args]) # noqa C419 # pylint: disable=use-a-generator -def activate(p, extra_network_data=None, step=0): +def activate(p, extra_network_data=None, step=0, include=[], exclude=[]): """call activate for extra networks in extra_network_data in specified order, then call activate for all remaining registered networks with an empty argument list""" if p.disable_extra_networks: return @@ -89,25 +90,29 @@ def activate(p, extra_network_data=None, step=0): shared.log.warning("Composable LoRA not compatible with 'lora_force_diffusers'") stepwise = False shared.opts.data['lora_functional'] = stepwise or functional - with devices.autocast(): - for extra_network_name, extra_network_args in extra_network_data.items(): - extra_network = extra_network_registry.get(extra_network_name, None) - if extra_network is None: - errors.log.warning(f"Skipping unknown extra network: {extra_network_name}") - continue - try: + + for extra_network_name, extra_network_args in extra_network_data.items(): + extra_network = extra_network_registry.get(extra_network_name, None) + if extra_network is None: + errors.log.warning(f"Skipping unknown extra network: {extra_network_name}") + continue + try: + signature = list(inspect.signature(extra_network.activate).parameters) + if 'include' in signature and 'exclude' in signature: + extra_network.activate(p, extra_network_args, step=step, include=include, exclude=exclude) + else: extra_network.activate(p, extra_network_args, step=step) - except Exception as e: - errors.display(e, f"Activating network: type={extra_network_name} args:{extra_network_args}") - - for extra_network_name, extra_network in extra_network_registry.items(): - args = extra_network_data.get(extra_network_name, None) - if args is not None: - continue - try: - extra_network.activate(p, []) - except Exception as e: - errors.display(e, f"Activating network: type={extra_network_name}") + except Exception as e: + errors.display(e, f"Activating network: type={extra_network_name} args:{extra_network_args}") + + for extra_network_name, extra_network in extra_network_registry.items(): + args = extra_network_data.get(extra_network_name, None) + if args is not None: + continue + try: + extra_network.activate(p, []) + except Exception as e: + errors.display(e, f"Activating network: type={extra_network_name}") p.network_data = extra_network_data if stepwise: diff --git a/modules/lora/extra_networks_lora.py b/modules/lora/extra_networks_lora.py index 4ce7a94a9..135df1ccb 100644 --- a/modules/lora/extra_networks_lora.py +++ b/modules/lora/extra_networks_lora.py @@ -112,7 +112,7 @@ def __init__(self): self.model = None self.errors = {} - def activate(self, p, params_list, step=0): + def activate(self, p, params_list, step=0, include=[], exclude=[]): self.errors.clear() if self.active: if self.model != shared.opts.sd_model_checkpoint: # reset if model changed @@ -123,8 +123,8 @@ def activate(self, p, params_list, step=0): self.model = shared.opts.sd_model_checkpoint names, te_multipliers, unet_multipliers, dyn_dims = parse(p, params_list, step) networks.network_load(names, te_multipliers, unet_multipliers, dyn_dims) # load - networks.network_activate() - if len(networks.loaded_networks) > 0 and step == 0: + networks.network_activate(include, exclude) + if len(networks.loaded_networks) > 0 and len(networks.applied_layers) > 0 and step == 0: infotext(p) prompt(p) shared.log.info(f'Load network: type=LoRA apply={[n.name for n in networks.loaded_networks]} te={te_multipliers} unet={unet_multipliers} time={networks.timer.summary}') diff --git a/modules/lora/networks.py b/modules/lora/networks.py index 805b24b52..edd82f3e4 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -19,6 +19,7 @@ available_networks = {} available_network_aliases = {} loaded_networks: List[network.Network] = [] +applied_layers: list[str] = [] bnb = None lora_cache = {} diffuser_loaded = [] @@ -465,7 +466,7 @@ def network_deactivate(): task = None pbar = nullcontext() with devices.inference_context(), pbar: - applied_layers = [] + applied_layers.clear() weights_devices = [] weights_dtypes = [] for component in modules.keys(): @@ -498,7 +499,7 @@ def network_deactivate(): sd_models.set_diffuser_offload(sd_model, op="model") -def network_activate(): +def network_activate(include=[], exclude=[]): t0 = time.time() timer.clear(complete=True) sd_model = getattr(shared.sd_model, "pipe", shared.sd_model) # wrapped model compatiblility @@ -506,10 +507,12 @@ def network_activate(): sd_models.disable_offload(sd_model) sd_models.move_model(sd_model, device=devices.cpu) modules = {} - for component_name in ['text_encoder','text_encoder_2', 'unet', 'transformer']: - component = getattr(sd_model, component_name, None) + components = include if len(include) > 0 else ['text_encoder', 'text_encoder_2', 'text_encoder_3', 'unet', 'transformer'] + components = [x for x in components if x not in exclude] + for name in components: + component = getattr(sd_model, name, None) if component is not None and hasattr(component, 'named_modules'): - modules[component_name] = list(component.named_modules()) + modules[name] = list(component.named_modules()) total = sum(len(x) for x in modules.values()) if len(loaded_networks) > 0: pbar = rp.Progress(rp.TextColumn('[cyan]Network: type=LoRA action=activate'), rp.BarColumn(), rp.TaskProgressColumn(), rp.TimeRemainingColumn(), rp.TimeElapsedColumn(), rp.TextColumn('[cyan]{task.description}'), console=shared.console) @@ -519,7 +522,7 @@ def network_activate(): pbar = nullcontext() with devices.inference_context(), pbar: wanted_names = tuple((x.name, x.te_multiplier, x.unet_multiplier, x.dyn_dim) for x in loaded_networks) if len(loaded_networks) > 0 else () - applied_layers = [] + applied_layers.clear() backup_size = 0 weights_devices = [] weights_dtypes = [] @@ -546,10 +549,12 @@ def network_activate(): module.network_current_names = wanted_names if task is not None: pbar.update(task, advance=1, description=f'networks={len(loaded_networks)} modules={total} apply={len(applied_layers)} backup={backup_size}') + if task is not None and len(applied_layers) == 0: + pbar.remove_task(task) # hide progress bar for no action weights_devices, weights_dtypes = list(set([x for x in weights_devices if x is not None])), list(set([x for x in weights_dtypes if x is not None])) # noqa: C403 # pylint: disable=R1718 timer.activate = time.time() - t0 if debug and len(loaded_networks) > 0: - shared.log.debug(f'Load network: type=LoRA networks={len(loaded_networks)} modules={total} apply={len(applied_layers)} device={weights_devices} dtype={weights_dtypes} backup={backup_size} fuse={shared.opts.lora_fuse_diffusers} time={timer.summary}') + shared.log.debug(f'Load network: type=LoRA networks={len(loaded_networks)} components={components} modules={total} apply={len(applied_layers)} device={weights_devices} dtype={weights_dtypes} backup={backup_size} fuse={shared.opts.lora_fuse_diffusers} time={timer.summary}') modules.clear() if shared.opts.diffusers_offload_mode == "sequential": sd_models.set_diffuser_offload(sd_model, op="model") diff --git a/modules/processing_args.py b/modules/processing_args.py index 93b0bf9b2..e7f53ba8e 100644 --- a/modules/processing_args.py +++ b/modules/processing_args.py @@ -6,7 +6,7 @@ import inspect import torch import numpy as np -from modules import shared, errors, sd_models, processing, processing_vae, processing_helpers, sd_hijack_hypertile, prompt_parser_diffusers, timer +from modules import shared, errors, sd_models, processing, processing_vae, processing_helpers, sd_hijack_hypertile, prompt_parser_diffusers, timer, extra_networks from modules.processing_callbacks import diffusers_callback_legacy, diffusers_callback, set_callbacks_p from modules.processing_helpers import resize_hires, fix_prompts, calculate_base_steps, calculate_hires_steps, calculate_refiner_steps, get_generator, set_latents, apply_circular # pylint: disable=unused-import from modules.api import helpers @@ -134,6 +134,7 @@ def set_pipeline_args(p, model, prompts: list, negative_prompts: list, prompts_2 else: prompt_parser_diffusers.embedder = None + extra_networks.activate(p, include=['text_encoder', 'text_encoder_2', 'text_encoder_3']) if 'prompt' in possible: if 'OmniGen' in model.__class__.__name__: prompts = [p.replace('|image|', '<|image_1|>') for p in prompts] diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index d22a9de97..627eb281f 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -89,7 +89,7 @@ def process_base(p: processing.StableDiffusionProcessing): sd_models.move_model(shared.sd_model.unet, devices.device) if hasattr(shared.sd_model, 'transformer'): sd_models.move_model(shared.sd_model.transformer, devices.device) - extra_networks.activate(p) + extra_networks.activate(p, exclude=['text_encoder', 'text_encoder_2']) hidiffusion.apply(p, shared.sd_model_type) # if 'image' in base_args: # base_args['image'] = set_latents(p) From 22d95204767bb1c3f6e89f05f017f15c8a0ac623 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 10 Dec 2024 08:41:38 -0500 Subject: [PATCH 074/249] fix sd upscale Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 3 ++- scripts/sd_upscale.py | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3aaec73f..6775ef37e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Change Log for SD.Next -## Update for 2024-12-06 +## Update for 2024-12-10 ### New models and integrations @@ -96,6 +96,7 @@ - simplify img2img/inpaint/sketch canvas handling - fix prompt caching - fix xyz grid skip final pass +- fix sd upscale script ## Update for 2024-11-21 diff --git a/scripts/sd_upscale.py b/scripts/sd_upscale.py index 9c5a72204..7ac31b603 100644 --- a/scripts/sd_upscale.py +++ b/scripts/sd_upscale.py @@ -48,7 +48,7 @@ def run(self, p, _, overlap, upscaler_index, scale_factor): # pylint: disable=ar else: img = init_img devices.torch_gc() - grid = images.split_grid(img, tile_w=p.width, tile_h=p.height, overlap=overlap) + grid = images.split_grid(img, tile_w=init_img.width, tile_h=init_img.height, overlap=overlap) batch_size = p.batch_size upscale_count = p.n_iter p.n_iter = 1 @@ -61,7 +61,7 @@ def run(self, p, _, overlap, upscaler_index, scale_factor): # pylint: disable=ar batch_count = math.ceil(len(work) / batch_size) state.job_count = batch_count * upscale_count - log.info(f"SD upscale: images={len(work)} tile={len(grid.tiles[0][2])}x{len(grid.tiles)} batches={state.job_count}") + log.info(f"SD upscale: images={len(work)} tiles={len(grid.tiles)} batches={state.job_count}") result_images = [] for n in range(upscale_count): @@ -91,4 +91,5 @@ def run(self, p, _, overlap, upscaler_index, scale_factor): # pylint: disable=ar images.save_image(combined_image, p.outpath_samples, "", start_seed, p.prompt, opts.samples_format, info=initial_info, p=p) processed = Processed(p, result_images, seed, initial_info) + log.info(f"SD upscale: images={result_images}") return processed From 5dfe26e8a231d5c9ad1ec953f2de3c6db223faec Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 10 Dec 2024 10:20:00 -0500 Subject: [PATCH 075/249] reorg settings Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 1 + javascript/script.js | 2 +- javascript/sdnext.css | 6 +- modules/processing.py | 3 + modules/processing_class.py | 3 + modules/shared.py | 255 +++++++++++++++++++----------------- scripts/cogvideo.py | 31 ++--- 7 files changed, 161 insertions(+), 140 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6775ef37e..5412861a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -67,6 +67,7 @@ - control: hide preview column by default - control: optionn to hide input column - control: add stats + - settings: reorganized and simplified - browser -> server logging framework - add addtional themes: `black-reimagined`, thanks @Artheriax diff --git a/javascript/script.js b/javascript/script.js index 836d9b102..f943f4626 100644 --- a/javascript/script.js +++ b/javascript/script.js @@ -125,7 +125,7 @@ document.addEventListener('keydown', (e) => { let elem; if (e.key === 'Escape') elem = getUICurrentTabContent().querySelector('button[id$=_interrupt]'); if (e.key === 'Enter' && e.ctrlKey) elem = getUICurrentTabContent().querySelector('button[id$=_generate]'); - if (e.key === 'r' && e.ctrlKey) elem = getUICurrentTabContent().querySelector('button[id$=_reprocess]'); + if (e.key === 'i' && e.ctrlKey) elem = getUICurrentTabContent().querySelector('button[id$=_reprocess]'); if (e.key === ' ' && e.ctrlKey) elem = getUICurrentTabContent().querySelector('button[id$=_extra_networks_btn]'); if (e.key === 'n' && e.ctrlKey) elem = getUICurrentTabContent().querySelector('button[id$=_extra_networks_btn]'); if (e.key === 's' && e.ctrlKey) elem = getUICurrentTabContent().querySelector('button[id^=save_]'); diff --git a/javascript/sdnext.css b/javascript/sdnext.css index c5145c973..60d835cd4 100644 --- a/javascript/sdnext.css +++ b/javascript/sdnext.css @@ -149,18 +149,20 @@ div#extras_scale_to_tab div.form { flex-direction: row; } #settings>div.tab-content { flex: 10 0 75%; display: grid; } #settings>div.tab-content>div { border: none; padding: 0; } #settings>div.tab-content>div>div>div>div>div { flex-direction: unset; } -#settings>div.tab-nav { display: grid; grid-template-columns: repeat(auto-fill, .5em minmax(10em, 1fr)); flex: 1 0 auto; width: 12em; align-self: flex-start; gap: var(--spacing-xxl); } +#settings>div.tab-nav { display: grid; grid-template-columns: repeat(auto-fill, .5em minmax(10em, 1fr)); flex: 1 0 auto; width: 12em; align-self: flex-start; gap: 8px; } #settings>div.tab-nav button { display: block; border: none; text-align: left; white-space: initial; padding: 0; } #settings>div.tab-nav>#settings_show_all_pages { padding: var(--size-2) var(--size-4); } #settings .block.gradio-checkbox { margin: 0; width: auto; } #settings .dirtyable { gap: .5em; } #settings .dirtyable.hidden { display: none; } -#settings .modification-indicator { height: 1.2em; border-radius: 1em !important; padding: 0; width: 0; margin-right: 0.5em; } +#settings .modification-indicator { height: 1.2em; border-radius: 1em !important; padding: 0; width: 0; margin-right: 0.5em; border-left: inset; } #settings .modification-indicator:disabled { visibility: hidden; } #settings .modification-indicator.saved { background: var(--color-accent-soft); width: var(--spacing-sm); } #settings .modification-indicator.changed { background: var(--color-accent); width: var(--spacing-sm); } #settings .modification-indicator.changed.unsaved { background-image: linear-gradient(var(--color-accent) 25%, var(--color-accent-soft) 75%); width: var(--spacing-sm); } #settings_result { margin: 0 1.2em; } +#tab_settings .gradio-slider, #tab_settings .gradio-dropdown { width: 300px !important; max-width: 300px; } +#tab_settings textarea { max-width: 500px; } .licenses { display: block !important; } /* live preview */ diff --git a/modules/processing.py b/modules/processing.py index 7ae397538..b4839e402 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -118,6 +118,9 @@ def js(self): def infotext(self, p: StableDiffusionProcessing, index): return create_infotext(p, self.all_prompts, self.all_seeds, self.all_subseeds, comments=[], position_in_batch=index % self.batch_size, iteration=index // self.batch_size) + def __str___(self): + return f'{self.__class__.__name__}: {self.__dict__}' + def process_images(p: StableDiffusionProcessing) -> Processed: timer.process.reset() diff --git a/modules/processing_class.py b/modules/processing_class.py index 2cbc07cc2..7a7d9cd36 100644 --- a/modules/processing_class.py +++ b/modules/processing_class.py @@ -339,6 +339,9 @@ def sample(self, conditioning, unconditional_conditioning, seeds, subseeds, subs def close(self): self.sampler = None # pylint: disable=attribute-defined-outside-init + def __str__(self): + return f'{self.__class__.__name__}: {self.__dict__}' + class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing): def __init__(self, **kwargs): diff --git a/modules/shared.py b/modules/shared.py index 92c0388a4..4dd26d601 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -467,69 +467,103 @@ def get_default_modes(): startup_offload_mode, startup_cross_attention, startup_sdp_options = get_default_modes() -options_templates.update(options_section(('sd', "Execution & Models"), { +options_templates.update(options_section(('sd', "Models & Loading"), { "sd_backend": OptionInfo(default_backend, "Execution backend", gr.Radio, {"choices": ["diffusers", "original"] }), + "diffusers_pipeline": OptionInfo('Autodetect', 'Model pipeline', gr.Dropdown, lambda: {"choices": list(shared_items.get_pipelines()), "visible": native}), "sd_model_checkpoint": OptionInfo(default_checkpoint, "Base model", DropdownEditable, lambda: {"choices": list_checkpoint_titles()}, refresh=refresh_checkpoints), "sd_model_refiner": OptionInfo('None', "Refiner model", gr.Dropdown, lambda: {"choices": ['None'] + list_checkpoint_titles()}, refresh=refresh_checkpoints), - "sd_vae": OptionInfo("Automatic", "VAE model", gr.Dropdown, lambda: {"choices": shared_items.sd_vae_items()}, refresh=shared_items.refresh_vae_list), "sd_unet": OptionInfo("None", "UNET model", gr.Dropdown, lambda: {"choices": shared_items.sd_unet_items()}, refresh=shared_items.refresh_unet_list), - "sd_text_encoder": OptionInfo('None', "Text encoder model", gr.Dropdown, lambda: {"choices": shared_items.sd_te_items()}, refresh=shared_items.refresh_te_list), - "sd_model_dict": OptionInfo('None', "Use separate base dict", gr.Dropdown, lambda: {"choices": ['None'] + list_checkpoint_titles()}, refresh=refresh_checkpoints), + "latent_history": OptionInfo(16, "Latent history size", gr.Slider, {"minimum": 1, "maximum": 100, "step": 1}), + + "offload_sep": OptionInfo("

Model Offloading

", "", gr.HTML), + "diffusers_move_base": OptionInfo(False, "Move base model to CPU when using refiner", gr.Checkbox, {"visible": False }), + "diffusers_move_unet": OptionInfo(False, "Move base model to CPU when using VAE", gr.Checkbox, {"visible": False }), + "diffusers_move_refiner": OptionInfo(False, "Move refiner model to CPU when not in use", gr.Checkbox, {"visible": False }), + "diffusers_extract_ema": OptionInfo(False, "Use model EMA weights when possible", gr.Checkbox, {"visible": False }), + "diffusers_offload_mode": OptionInfo(startup_offload_mode, "Model offload mode", gr.Radio, {"choices": ['none', 'balanced', 'model', 'sequential']}), + "diffusers_offload_min_gpu_memory": OptionInfo(0.25, "Balanced offload GPU low watermark", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01 }), + "diffusers_offload_max_gpu_memory": OptionInfo(0.70, "Balanced offload GPU high watermark", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01 }), + "diffusers_offload_max_cpu_memory": OptionInfo(0.90, "Balanced offload CPU high watermark", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01 }), + + "advanced_sep": OptionInfo("

Advanced Options

", "", gr.HTML), "sd_checkpoint_autoload": OptionInfo(True, "Model autoload on start"), "sd_checkpoint_autodownload": OptionInfo(True, "Model auto-download on demand"), - "sd_textencoder_cache": OptionInfo(True, "Cache text encoder results", gr.Checkbox, {"visible": False}), - "sd_textencoder_cache_size": OptionInfo(4, "Text encoder cache size", gr.Slider, {"minimum": 0, "maximum": 16, "step": 1}), "stream_load": OptionInfo(False, "Load models using stream loading method", gr.Checkbox, {"visible": not native }), + "diffusers_eval": OptionInfo(True, "Force model eval", gr.Checkbox, {"visible": False }), + "diffusers_to_gpu": OptionInfo(False, "Load model directly to GPU"), + "disable_accelerate": OptionInfo(False, "Disable accelerate", gr.Checkbox, {"visible": False }), + "sd_model_dict": OptionInfo('None', "Use separate base dict", gr.Dropdown, lambda: {"choices": ['None'] + list_checkpoint_titles()}, refresh=refresh_checkpoints), + "sd_checkpoint_cache": OptionInfo(0, "Cached models", gr.Slider, {"minimum": 0, "maximum": 10, "step": 1, "visible": not native }), +})) + +options_templates.update(options_section(('vae_encoder', "Variable Auto Encoder"), { + "sd_vae": OptionInfo("Automatic", "VAE model", gr.Dropdown, lambda: {"choices": shared_items.sd_vae_items()}, refresh=shared_items.refresh_vae_list), + "diffusers_vae_upcast": OptionInfo("default", "VAE upcasting", gr.Radio, {"choices": ['default', 'true', 'false']}), + "no_half_vae": OptionInfo(False if not cmd_opts.use_openvino else True, "Full precision (--no-half-vae)"), + "diffusers_vae_slicing": OptionInfo(True, "VAE slicing", gr.Checkbox, {"visible": native}), + "diffusers_vae_tiling": OptionInfo(cmd_opts.lowvram or cmd_opts.medvram, "VAE tiling", gr.Checkbox, {"visible": native}), + "sd_vae_sliced_encode": OptionInfo(False, "VAE sliced encode", gr.Checkbox, {"visible": not native}), + "nan_skip": OptionInfo(False, "Skip Generation if NaN found in latents", gr.Checkbox), + "rollback_vae": OptionInfo(False, "Attempt VAE roll back for NaN values"), +})) + +options_templates.update(options_section(('text_encoder', "Text Encoder"), { + "sd_text_encoder": OptionInfo('None', "Text encoder model", gr.Dropdown, lambda: {"choices": shared_items.sd_te_items()}, refresh=shared_items.refresh_te_list), + "prompt_attention": OptionInfo("native", "Prompt attention parser", gr.Radio, {"choices": ["native", "compel", "xhinker", "a1111", "fixed"] }), "prompt_mean_norm": OptionInfo(False, "Prompt attention normalization", gr.Checkbox), + "sd_textencoder_cache": OptionInfo(True, "Cache text encoder results", gr.Checkbox, {"visible": False}), + "sd_textencoder_cache_size": OptionInfo(4, "Text encoder cache size", gr.Slider, {"minimum": 0, "maximum": 16, "step": 1}), "comma_padding_backtrack": OptionInfo(20, "Prompt padding", gr.Slider, {"minimum": 0, "maximum": 74, "step": 1, "visible": not native }), - "prompt_attention": OptionInfo("native", "Prompt attention parser", gr.Radio, {"choices": ["native", "compel", "xhinker", "a1111", "fixed"] }), - "latent_history": OptionInfo(16, "Latent history size", gr.Slider, {"minimum": 1, "maximum": 100, "step": 1}), - "sd_checkpoint_cache": OptionInfo(0, "Cached models", gr.Slider, {"minimum": 0, "maximum": 10, "step": 1, "visible": not native }), + "diffusers_zeros_prompt_pad": OptionInfo(False, "Use zeros for prompt padding", gr.Checkbox), + "diffusers_pooled": OptionInfo("default", "Diffusers SDXL pooled embeds", gr.Radio, {"choices": ['default', 'weighted']}), })) options_templates.update(options_section(('cuda', "Compute Settings"), { - "math_sep": OptionInfo("

Execution precision

", "", gr.HTML), + "math_sep": OptionInfo("

Execution Precision

", "", gr.HTML), "precision": OptionInfo("Autocast", "Precision type", gr.Radio, {"choices": ["Autocast", "Full"]}), "cuda_dtype": OptionInfo("Auto", "Device precision type", gr.Radio, {"choices": ["Auto", "FP32", "FP16", "BF16"]}), + "no_half": OptionInfo(False if not cmd_opts.use_openvino else True, "Full precision (--no-half)", None, None, None), + "upcast_sampling": OptionInfo(False if sys.platform != "darwin" else True, "Upcast sampling", gr.Checkbox, {"visible": not native}), + "upcast_attn": OptionInfo(False, "Upcast attention layer", gr.Checkbox, {"visible": not native}), + "cuda_cast_unet": OptionInfo(False, "Fixed UNet precision", gr.Checkbox, {"visible": not native}), - "model_sep": OptionInfo("

Model options

", "", gr.HTML), - "no_half": OptionInfo(False if not cmd_opts.use_openvino else True, "Full precision for model (--no-half)", None, None, None), - "no_half_vae": OptionInfo(False if not cmd_opts.use_openvino else True, "Full precision for VAE (--no-half-vae)"), - "upcast_sampling": OptionInfo(False if sys.platform != "darwin" else True, "Upcast sampling"), - "upcast_attn": OptionInfo(False, "Upcast attention layer"), - "cuda_cast_unet": OptionInfo(False, "Fixed UNet precision"), - "nan_skip": OptionInfo(False, "Skip Generation if NaN found in latents", gr.Checkbox), - "rollback_vae": OptionInfo(False, "Attempt VAE roll back for NaN values"), + "generator_sep": OptionInfo("

Noise Options

", "", gr.HTML), + "diffusers_generator_device": OptionInfo("GPU", "Generator device", gr.Radio, {"choices": ["GPU", "CPU", "Unset"]}), "cross_attention_sep": OptionInfo("

Cross Attention

", "", gr.HTML), - "cross_attention_optimization": OptionInfo(startup_cross_attention, "Attention optimization method", gr.Radio, lambda: {"choices": shared_items.list_crossattention(native) }), - "sdp_options": OptionInfo(startup_sdp_options, "SDP options", gr.CheckboxGroup, {"choices": ['Flash attention', 'Memory attention', 'Math attention', 'Dynamic attention', 'Sage attention'] }), + "cross_attention_optimization": OptionInfo(startup_cross_attention, "Attention optimization method", gr.Radio, lambda: {"choices": shared_items.list_crossattention(native)}), + "sdp_options": OptionInfo(startup_sdp_options, "SDP options", gr.CheckboxGroup, {"choices": ['Flash attention', 'Memory attention', 'Math attention', 'Dynamic attention', 'Sage attention'], "visible": native}), "xformers_options": OptionInfo(['Flash attention'], "xFormers options", gr.CheckboxGroup, {"choices": ['Flash attention'] }), "dynamic_attention_slice_rate": OptionInfo(4, "Dynamic Attention slicing rate in GB", gr.Slider, {"minimum": 0.1, "maximum": gpu_memory, "step": 0.1, "visible": native}), "sub_quad_sep": OptionInfo("

Sub-quadratic options

", "", gr.HTML, {"visible": not native}), "sub_quad_q_chunk_size": OptionInfo(512, "Attention query chunk size", gr.Slider, {"minimum": 16, "maximum": 8192, "step": 8, "visible": not native}), "sub_quad_kv_chunk_size": OptionInfo(512, "Attention kv chunk size", gr.Slider, {"minimum": 0, "maximum": 8192, "step": 8, "visible": not native}), "sub_quad_chunk_threshold": OptionInfo(80, "Attention chunking threshold", gr.Slider, {"minimum": 0, "maximum": 100, "step": 1, "visible": not native}), +})) - "other_sep": OptionInfo("

Execution options

", "", gr.HTML), - "opt_channelslast": OptionInfo(False, "Use channels last "), - "cudnn_deterministic": OptionInfo(False, "Use deterministic mode"), - "cudnn_benchmark": OptionInfo(False, "Full-depth cuDNN benchmark feature"), +options_templates.update(options_section(('backends', "Backend Settings"), { + "other_sep": OptionInfo("

Torch Options

", "", gr.HTML), + "opt_channelslast": OptionInfo(False, "Channels last "), + "cudnn_deterministic": OptionInfo(False, "Deterministic mode"), + "cudnn_benchmark": OptionInfo(False, "Full-depth cuDNN benchmark"), "diffusers_fuse_projections": OptionInfo(False, "Fused projections"), - "torch_expandable_segments": OptionInfo(False, "Torch expandable segments"), - "cuda_mem_fraction": OptionInfo(0.0, "Torch memory limit", gr.Slider, {"minimum": 0, "maximum": 2.0, "step": 0.05}), - "torch_gc_threshold": OptionInfo(80, "Torch memory threshold for GC", gr.Slider, {"minimum": 0, "maximum": 100, "step": 1}), - "torch_malloc": OptionInfo("native", "Torch memory allocator", gr.Radio, {"choices": ['native', 'cudaMallocAsync'] }), + "torch_expandable_segments": OptionInfo(False, "Expandable segments"), + "cuda_mem_fraction": OptionInfo(0.0, "Memory limit", gr.Slider, {"minimum": 0, "maximum": 2.0, "step": 0.05}), + "torch_gc_threshold": OptionInfo(80, "GC threshold", gr.Slider, {"minimum": 0, "maximum": 100, "step": 1}), + "inference_mode": OptionInfo("no-grad", "Inference mode", gr.Radio, {"choices": ["no-grad", "inference-mode", "none"]}), + "torch_malloc": OptionInfo("native", "Memory allocator", gr.Radio, {"choices": ['native', 'cudaMallocAsync'] }), + + "onnx_sep": OptionInfo("

ONNX

", "", gr.HTML), + "onnx_execution_provider": OptionInfo(execution_providers.get_default_execution_provider().value, 'ONNX Execution Provider', gr.Dropdown, lambda: {"choices": execution_providers.available_execution_providers }), + "onnx_cpu_fallback": OptionInfo(True, 'ONNX allow fallback to CPU'), + "onnx_cache_converted": OptionInfo(True, 'ONNX cache converted models'), + "onnx_unload_base": OptionInfo(False, 'ONNX unload base model when processing refiner'), - "cuda_compile_sep": OptionInfo("

Model Compile

", "", gr.HTML), - "cuda_compile": OptionInfo([] if not cmd_opts.use_openvino else ["Model", "VAE"], "Compile Model", gr.CheckboxGroup, {"choices": ["Model", "VAE", "Text Encoder", "Upscaler"]}), - "cuda_compile_backend": OptionInfo("none" if not cmd_opts.use_openvino else "openvino_fx", "Model compile backend", gr.Radio, {"choices": ['none', 'inductor', 'cudagraphs', 'aot_ts_nvfuser', 'hidet', 'migraphx', 'ipex', 'onediff', 'stable-fast', 'deep-cache', 'olive-ai', 'openvino_fx']}), - "cuda_compile_mode": OptionInfo("default", "Model compile mode", gr.Radio, {"choices": ['default', 'reduce-overhead', 'max-autotune', 'max-autotune-no-cudagraphs']}), - "cuda_compile_fullgraph": OptionInfo(True if not cmd_opts.use_openvino else False, "Model compile fullgraph"), - "cuda_compile_precompile": OptionInfo(False, "Model compile precompile"), - "cuda_compile_verbose": OptionInfo(False, "Model compile verbose mode"), - "cuda_compile_errors": OptionInfo(True, "Model compile suppress errors"), - "deep_cache_interval": OptionInfo(3, "DeepCache cache interval", gr.Slider, {"minimum": 1, "maximum": 10, "step": 1}), + "olive_sep": OptionInfo("

Olive

", "", gr.HTML), + "olive_float16": OptionInfo(True, 'Olive use FP16 on optimization'), + "olive_vae_encoder_float32": OptionInfo(False, 'Olive force FP32 for VAE Encoder'), + "olive_static_dims": OptionInfo(True, 'Olive use static dimensions'), + "olive_cache_optimized": OptionInfo(True, 'Olive cache optimized models'), "ipex_sep": OptionInfo("

IPEX

", "", gr.HTML, {"visible": devices.backend == "ipex"}), "ipex_optimize": OptionInfo([], "IPEX Optimize for Intel GPUs", gr.CheckboxGroup, {"choices": ["Model", "VAE", "Text Encoder", "Upscaler"], "visible": devices.backend == "ipex"}), @@ -543,91 +577,55 @@ def get_default_modes(): "directml_sep": OptionInfo("

DirectML

", "", gr.HTML, {"visible": devices.backend == "directml"}), "directml_memory_provider": OptionInfo(default_memory_provider, 'DirectML memory stats provider', gr.Radio, {"choices": memory_providers, "visible": devices.backend == "directml"}), "directml_catch_nan": OptionInfo(False, "DirectML retry ops for NaN", gr.Checkbox, {"visible": devices.backend == "directml"}), - - "olive_sep": OptionInfo("

Olive

", "", gr.HTML), - "olive_float16": OptionInfo(True, 'Olive use FP16 on optimization'), - "olive_vae_encoder_float32": OptionInfo(False, 'Olive force FP32 for VAE Encoder'), - "olive_static_dims": OptionInfo(True, 'Olive use static dimensions'), - "olive_cache_optimized": OptionInfo(True, 'Olive cache optimized models'), -})) - -options_templates.update(options_section(('diffusers', "Diffusers Settings"), { - "diffusers_pipeline": OptionInfo('Autodetect', 'Diffusers pipeline', gr.Dropdown, lambda: {"choices": list(shared_items.get_pipelines()) }), - "diffuser_cache_config": OptionInfo(True, "Use cached model config when available"), - "diffusers_move_base": OptionInfo(False, "Move base model to CPU when using refiner"), - "diffusers_move_unet": OptionInfo(False, "Move base model to CPU when using VAE"), - "diffusers_move_refiner": OptionInfo(False, "Move refiner model to CPU when not in use"), - "diffusers_extract_ema": OptionInfo(False, "Use model EMA weights when possible"), - "diffusers_generator_device": OptionInfo("GPU", "Generator device", gr.Radio, {"choices": ["GPU", "CPU", "Unset"]}), - "diffusers_offload_mode": OptionInfo(startup_offload_mode, "Model offload mode", gr.Radio, {"choices": ['none', 'balanced', 'model', 'sequential']}), - "diffusers_offload_min_gpu_memory": OptionInfo(0.25, "Balanced offload GPU low watermark", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01 }), - "diffusers_offload_max_gpu_memory": OptionInfo(0.70, "Balanced offload GPU high watermark", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01 }), - "diffusers_offload_max_cpu_memory": OptionInfo(0.75, "Balanced offload CPU high watermark", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01 }), - "diffusers_vae_upcast": OptionInfo("default", "VAE upcasting", gr.Radio, {"choices": ['default', 'true', 'false']}), - "diffusers_vae_slicing": OptionInfo(True, "VAE slicing"), - "diffusers_vae_tiling": OptionInfo(cmd_opts.lowvram or cmd_opts.medvram, "VAE tiling"), - "diffusers_model_load_variant": OptionInfo("default", "Preferred Model variant", gr.Radio, {"choices": ['default', 'fp32', 'fp16']}), - "diffusers_vae_load_variant": OptionInfo("default", "Preferred VAE variant", gr.Radio, {"choices": ['default', 'fp32', 'fp16']}), - "custom_diffusers_pipeline": OptionInfo('', 'Load custom Diffusers pipeline'), - "diffusers_eval": OptionInfo(True, "Force model eval"), - "diffusers_to_gpu": OptionInfo(False, "Load model directly to GPU"), - "disable_accelerate": OptionInfo(False, "Disable accelerate"), - "diffusers_pooled": OptionInfo("default", "Diffusers SDXL pooled embeds", gr.Radio, {"choices": ['default', 'weighted']}), - "diffusers_zeros_prompt_pad": OptionInfo(False, "Use zeros for prompt padding", gr.Checkbox), - "huggingface_token": OptionInfo('', 'HuggingFace token'), - "enable_linfusion": OptionInfo(False, "Apply LinFusion distillation on load"), - - "onnx_sep": OptionInfo("

ONNX Runtime

", "", gr.HTML), - "onnx_execution_provider": OptionInfo(execution_providers.get_default_execution_provider().value, 'Execution Provider', gr.Dropdown, lambda: {"choices": execution_providers.available_execution_providers }), - "onnx_cpu_fallback": OptionInfo(True, 'ONNX allow fallback to CPU'), - "onnx_cache_converted": OptionInfo(True, 'ONNX cache converted models'), - "onnx_unload_base": OptionInfo(False, 'ONNX unload base model when processing refiner'), })) options_templates.update(options_section(('quantization', "Quantization Settings"), { - "bnb_quantization": OptionInfo([], "BnB quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "VAE", "Text Encoder"], "visible": native}), - "bnb_quantization_type": OptionInfo("nf4", "BnB quantization type", gr.Radio, {"choices": ['nf4', 'fp8', 'fp4'], "visible": native}), - "bnb_quantization_storage": OptionInfo("uint8", "BnB quantization storage", gr.Radio, {"choices": ["float16", "float32", "int8", "uint8", "float64", "bfloat16"], "visible": native}), - "optimum_quanto_weights": OptionInfo([], "Optimum.quanto quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "VAE", "Text Encoder", "ControlNet"], "visible": native}), - "optimum_quanto_weights_type": OptionInfo("qint8", "Optimum.quanto quantization type", gr.Radio, {"choices": ['qint8', 'qfloat8_e4m3fn', 'qfloat8_e5m2', 'qint4', 'qint2'], "visible": native}), - "optimum_quanto_activations_type": OptionInfo("none", "Optimum.quanto quantization activations ", gr.Radio, {"choices": ['none', 'qint8', 'qfloat8_e4m3fn', 'qfloat8_e5m2'], "visible": native}), - "torchao_quantization": OptionInfo([], "TorchAO quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "VAE", "Text Encoder"], "visible": native}), - "torchao_quantization_mode": OptionInfo("pre", "TorchAO quantization mode", gr.Radio, {"choices": ['pre', 'post'], "visible": native}), - "torchao_quantization_type": OptionInfo("int8", "TorchAO quantization type", gr.Radio, {"choices": ["int8+act", "int8", "int4", "fp8+act", "fp8", "fpx"], "visible": native}), - "nncf_compress_weights": OptionInfo([], "NNCF compression enabled", gr.CheckboxGroup, {"choices": ["Model", "VAE", "Text Encoder", "ControlNet"], "visible": native}), - "nncf_compress_weights_mode": OptionInfo("INT8", "NNCF compress mode", gr.Radio, {"choices": ['INT8', 'INT8_SYM', 'INT4_ASYM', 'INT4_SYM', 'NF4'] if cmd_opts.use_openvino else ['INT8']}), - "nncf_compress_weights_raito": OptionInfo(1.0, "NNCF compress ratio", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01, "visible": cmd_opts.use_openvino}), - "nncf_quantize": OptionInfo([], "NNCF OpenVINO quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "VAE", "Text Encoder"], "visible": cmd_opts.use_openvino}), - "nncf_quant_mode": OptionInfo("INT8", "NNCF OpenVINO quantization mode", gr.Radio, {"choices": ['INT8', 'FP8_E4M3', 'FP8_E5M2'], "visible": cmd_opts.use_openvino}), - - "quant_shuffle_weights": OptionInfo(False, "Shuffle the weights between GPU and CPU when quantizing", gr.Checkbox, {"visible": native}), + "bnb_sep": OptionInfo("

BitsAndBytes

", "", gr.HTML), + "bnb_quantization": OptionInfo([], "Enabled", gr.CheckboxGroup, {"choices": ["Model", "VAE", "Text Encoder"], "visible": native}), + "bnb_quantization_type": OptionInfo("nf4", "Type", gr.Radio, {"choices": ['nf4', 'fp8', 'fp4'], "visible": native}), + "bnb_quantization_storage": OptionInfo("uint8", "Backend storage", gr.Radio, {"choices": ["float16", "float32", "int8", "uint8", "float64", "bfloat16"], "visible": native}), + "optimum_quanto_sep": OptionInfo("

Optimum Quanto

", "", gr.HTML), + "optimum_quanto_weights": OptionInfo([], "Enabled", gr.CheckboxGroup, {"choices": ["Model", "VAE", "Text Encoder", "ControlNet"], "visible": native}), + "optimum_quanto_weights_type": OptionInfo("qint8", "Type", gr.Radio, {"choices": ['qint8', 'qfloat8_e4m3fn', 'qfloat8_e5m2', 'qint4', 'qint2'], "visible": native}), + "optimum_quanto_activations_type": OptionInfo("none", "Activations ", gr.Radio, {"choices": ['none', 'qint8', 'qfloat8_e4m3fn', 'qfloat8_e5m2'], "visible": native}), + "torchao_sep": OptionInfo("

TorchAO

", "", gr.HTML), + "torchao_quantization": OptionInfo([], "Enabled", gr.CheckboxGroup, {"choices": ["Model", "VAE", "Text Encoder"], "visible": native}), + "torchao_quantization_mode": OptionInfo("pre", "Mode", gr.Radio, {"choices": ['pre', 'post'], "visible": native}), + "torchao_quantization_type": OptionInfo("int8", "Type", gr.Radio, {"choices": ["int8+act", "int8", "int4", "fp8+act", "fp8", "fpx"], "visible": native}), + "nncf_sep": OptionInfo("

NNCF

", "", gr.HTML), + "nncf_compress_weights": OptionInfo([], "Enabled", gr.CheckboxGroup, {"choices": ["Model", "VAE", "Text Encoder", "ControlNet"], "visible": native}), + "nncf_compress_weights_mode": OptionInfo("INT8", "Mode", gr.Radio, {"choices": ['INT8', 'INT8_SYM', 'INT4_ASYM', 'INT4_SYM', 'NF4'] if cmd_opts.use_openvino else ['INT8']}), + "nncf_compress_weights_raito": OptionInfo(1.0, "Compress ratio", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01, "visible": cmd_opts.use_openvino}), + "nncf_quantize": OptionInfo([], "OpenVINO enabled", gr.CheckboxGroup, {"choices": ["Model", "VAE", "Text Encoder"], "visible": cmd_opts.use_openvino}), + "nncf_quant_mode": OptionInfo("INT8", "OpenVINO mode", gr.Radio, {"choices": ['INT8', 'FP8_E4M3', 'FP8_E5M2'], "visible": cmd_opts.use_openvino}), + "quant_shuffle_weights": OptionInfo(False, "Shuffle weights", gr.Checkbox, {"visible": native}), })) -options_templates.update(options_section(('advanced', "Inference Settings"), { - "token_merging_sep": OptionInfo("

Token merging

", "", gr.HTML), +options_templates.update(options_section(('advanced', "Pipeline Modifiers"), { + "token_merging_sep": OptionInfo("

Token Merging

", "", gr.HTML), "token_merging_method": OptionInfo("None", "Token merging method", gr.Radio, {"choices": ['None', 'ToMe', 'ToDo']}), "tome_ratio": OptionInfo(0.0, "ToMe token merging ratio", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.05}), "todo_ratio": OptionInfo(0.0, "ToDo token merging ratio", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.05}), "freeu_sep": OptionInfo("

FreeU

", "", gr.HTML), "freeu_enabled": OptionInfo(False, "FreeU"), - "freeu_b1": OptionInfo(1.2, "1st stage backbone factor", gr.Slider, {"minimum": 1.0, "maximum": 2.0, "step": 0.01}), - "freeu_b2": OptionInfo(1.4, "2nd stage backbone factor", gr.Slider, {"minimum": 1.0, "maximum": 2.0, "step": 0.01}), - "freeu_s1": OptionInfo(0.9, "1st stage skip factor", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}), - "freeu_s2": OptionInfo(0.2, "2nd stage skip factor", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}), + "freeu_b1": OptionInfo(1.2, "1st stage backbone", gr.Slider, {"minimum": 1.0, "maximum": 2.0, "step": 0.01}), + "freeu_b2": OptionInfo(1.4, "2nd stage backbone", gr.Slider, {"minimum": 1.0, "maximum": 2.0, "step": 0.01}), + "freeu_s1": OptionInfo(0.9, "1st stage skip", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}), + "freeu_s2": OptionInfo(0.2, "2nd stage skip", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}), "pag_sep": OptionInfo("

Perturbed-Attention Guidance

", "", gr.HTML), "pag_apply_layers": OptionInfo("m0", "PAG layer names"), "hypertile_sep": OptionInfo("

HyperTile

", "", gr.HTML), - "hypertile_hires_only": OptionInfo(False, "HyperTile hires pass only"), - "hypertile_unet_enabled": OptionInfo(False, "HyperTile UNet"), - "hypertile_unet_tile": OptionInfo(0, "HyperTile UNet tile size", gr.Slider, {"minimum": 0, "maximum": 1024, "step": 8}), - "hypertile_unet_swap_size": OptionInfo(1, "HyperTile UNet swap size", gr.Slider, {"minimum": 1, "maximum": 10, "step": 1}), - "hypertile_unet_depth": OptionInfo(0, "HyperTile UNet depth", gr.Slider, {"minimum": 0, "maximum": 4, "step": 1}), - "hypertile_vae_enabled": OptionInfo(False, "HyperTile VAE", gr.Checkbox), - "hypertile_vae_tile": OptionInfo(128, "HyperTile VAE tile size", gr.Slider, {"minimum": 0, "maximum": 1024, "step": 8}), - "hypertile_vae_swap_size": OptionInfo(1, "HyperTile VAE swap size", gr.Slider, {"minimum": 1, "maximum": 10, "step": 1}), + "hypertile_hires_only": OptionInfo(False, "HiRes pass only"), + "hypertile_unet_enabled": OptionInfo(False, "UNet Enabled"), + "hypertile_unet_tile": OptionInfo(0, "UNet tile size", gr.Slider, {"minimum": 0, "maximum": 1024, "step": 8}), + "hypertile_unet_swap_size": OptionInfo(1, "UNet swap size", gr.Slider, {"minimum": 1, "maximum": 10, "step": 1}), + "hypertile_unet_depth": OptionInfo(0, "UNet depth", gr.Slider, {"minimum": 0, "maximum": 4, "step": 1}), + "hypertile_vae_enabled": OptionInfo(False, "VAE Enabled", gr.Checkbox), + "hypertile_vae_tile": OptionInfo(128, "VAE tile size", gr.Slider, {"minimum": 0, "maximum": 1024, "step": 8}), + "hypertile_vae_swap_size": OptionInfo(1, "VAE swap size", gr.Slider, {"minimum": 1, "maximum": 10, "step": 1}), "hidiffusion_sep": OptionInfo("

HiDiffusion

", "", gr.HTML), "hidiffusion_raunet": OptionInfo(True, "Apply RAU-Net"), @@ -636,16 +634,28 @@ def get_default_modes(): "hidiffusion_t1": OptionInfo(-1, "Override T1 ratio", gr.Slider, {"minimum": -1, "maximum": 1.0, "step": 0.05}), "hidiffusion_t2": OptionInfo(-1, "Override T2 ratio", gr.Slider, {"minimum": -1, "maximum": 1.0, "step": 0.05}), + "linfusion_sep": OptionInfo("

Batch

", "", gr.HTML), + "enable_linfusion": OptionInfo(False, "Apply LinFusion distillation on load"), + "inference_batch_sep": OptionInfo("

Batch

", "", gr.HTML), "sequential_seed": OptionInfo(True, "Batch mode uses sequential seeds"), "batch_frame_mode": OptionInfo(False, "Parallel process images in batch"), - "inference_other_sep": OptionInfo("

Other

", "", gr.HTML), - "inference_mode": OptionInfo("no-grad", "Torch inference mode", gr.Radio, {"choices": ["no-grad", "inference-mode", "none"]}), - "sd_vae_sliced_encode": OptionInfo(False, "VAE sliced encode", gr.Checkbox, {"visible": not native}), +})) + +options_templates.update(options_section(('compile', "Model Compile"), { + "cuda_compile_sep": OptionInfo("

Model Compile

", "", gr.HTML), + "cuda_compile": OptionInfo([] if not cmd_opts.use_openvino else ["Model", "VAE"], "Compile Model", gr.CheckboxGroup, {"choices": ["Model", "VAE", "Text Encoder", "Upscaler"]}), + "cuda_compile_backend": OptionInfo("none" if not cmd_opts.use_openvino else "openvino_fx", "Model compile backend", gr.Radio, {"choices": ['none', 'inductor', 'cudagraphs', 'aot_ts_nvfuser', 'hidet', 'migraphx', 'ipex', 'onediff', 'stable-fast', 'deep-cache', 'olive-ai', 'openvino_fx']}), + "cuda_compile_mode": OptionInfo("default", "Model compile mode", gr.Radio, {"choices": ['default', 'reduce-overhead', 'max-autotune', 'max-autotune-no-cudagraphs']}), + "cuda_compile_fullgraph": OptionInfo(True if not cmd_opts.use_openvino else False, "Model compile fullgraph"), + "cuda_compile_precompile": OptionInfo(False, "Model compile precompile"), + "cuda_compile_verbose": OptionInfo(False, "Model compile verbose mode"), + "cuda_compile_errors": OptionInfo(True, "Model compile suppress errors"), + "deep_cache_interval": OptionInfo(3, "DeepCache cache interval", gr.Slider, {"minimum": 1, "maximum": 10, "step": 1}), })) options_templates.update(options_section(('system-paths', "System Paths"), { - "models_paths_sep_options": OptionInfo("

Models paths

", "", gr.HTML), + "models_paths_sep_options": OptionInfo("

Models Paths

", "", gr.HTML), "models_dir": OptionInfo('models', "Base path where all models are stored", folder=True), "ckpt_dir": OptionInfo(os.path.join(paths.models_path, 'Stable-diffusion'), "Folder with stable diffusion models", folder=True), "diffusers_dir": OptionInfo(os.path.join(paths.models_path, 'Diffusers'), "Folder with Huggingface models", folder=True), @@ -726,13 +736,13 @@ def get_default_modes(): })) options_templates.update(options_section(('saving-paths', "Image Paths"), { - "saving_sep_images": OptionInfo("

Save options

", "", gr.HTML), + "saving_sep_images": OptionInfo("

Save Options

", "", gr.HTML), "save_images_add_number": OptionInfo(True, "Numbered filenames", component_args=hide_dirs), "use_original_name_batch": OptionInfo(True, "Batch uses original name"), "save_to_dirs": OptionInfo(False, "Save images to a subdirectory"), "directories_filename_pattern": OptionInfo("[date]", "Directory name pattern", component_args=hide_dirs), "samples_filename_pattern": OptionInfo("[seq]-[model_name]-[prompt_words]", "Images filename pattern", component_args=hide_dirs), - "directories_max_prompt_words": OptionInfo(8, "Max words per pattern", gr.Slider, {"minimum": 1, "maximum": 99, "step": 1, **hide_dirs}), + "directories_max_prompt_words": OptionInfo(8, "Max words", gr.Slider, {"minimum": 1, "maximum": 99, "step": 1, **hide_dirs}), "outdir_sep_dirs": OptionInfo("

Folders

", "", gr.HTML), "outdir_samples": OptionInfo("", "Images folder", component_args=hide_dirs, folder=True), @@ -751,14 +761,14 @@ def get_default_modes(): "outdir_control_grids": OptionInfo("outputs/grids", 'Folder for control grids', component_args=hide_dirs, folder=True), })) -options_templates.update(options_section(('ui', "User Interface Options"), { +options_templates.update(options_section(('ui', "User Interface"), { "theme_type": OptionInfo("Standard", "Theme type", gr.Radio, {"choices": ["Modern", "Standard", "None"]}), "theme_style": OptionInfo("Auto", "Theme mode", gr.Radio, {"choices": ["Auto", "Dark", "Light"]}), "gradio_theme": OptionInfo("black-teal", "UI theme", gr.Dropdown, lambda: {"choices": theme.list_themes()}, refresh=theme.refresh_themes), "autolaunch": OptionInfo(False, "Autolaunch browser upon startup"), "font_size": OptionInfo(14, "Font size", gr.Slider, {"minimum": 8, "maximum": 32, "step": 1, "visible": True}), "aspect_ratios": OptionInfo("1:1, 4:3, 3:2, 16:9, 16:10, 21:9, 2:3, 3:4, 9:16, 10:16, 9:21", "Allowed aspect ratios"), - "motd": OptionInfo(True, "Show MOTD"), + "motd": OptionInfo(False, "Show MOTD"), "compact_view": OptionInfo(False, "Compact view"), "return_grid": OptionInfo(True, "Show grid in results"), "return_mask": OptionInfo(False, "Inpainting include greyscale mask in results"), @@ -770,14 +780,14 @@ def get_default_modes(): })) options_templates.update(options_section(('live-preview', "Live Previews"), { - "notification_audio_enable": OptionInfo(False, "Play a notification upon completion"), - "notification_audio_path": OptionInfo("html/notification.mp3","Path to notification sound", component_args=hide_dirs, folder=True), "show_progress_every_n_steps": OptionInfo(1, "Live preview display period", gr.Slider, {"minimum": 0, "maximum": 32, "step": 1}), "show_progress_type": OptionInfo("Approximate", "Live preview method", gr.Radio, {"choices": ["Simple", "Approximate", "TAESD", "Full VAE"]}), "live_preview_refresh_period": OptionInfo(500, "Progress update period", gr.Slider, {"minimum": 0, "maximum": 5000, "step": 25}), "live_preview_taesd_layers": OptionInfo(3, "TAESD decode layers", gr.Slider, {"minimum": 1, "maximum": 3, "step": 1}), "logmonitor_show": OptionInfo(True, "Show log view"), "logmonitor_refresh_period": OptionInfo(5000, "Log view update period", gr.Slider, {"minimum": 0, "maximum": 30000, "step": 25}), + "notification_audio_enable": OptionInfo(False, "Play a notification upon completion"), + "notification_audio_path": OptionInfo("html/notification.mp3","Path to notification sound", component_args=hide_dirs, folder=True), })) options_templates.update(options_section(('sampler-params', "Sampler Settings"), { @@ -816,7 +826,7 @@ def get_default_modes(): 's_noise': OptionInfo(1.0, "Sigma noise", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01, "visible": not native}), 's_min': OptionInfo(0.0, "Sigma min", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01, "visible": not native}), 's_max': OptionInfo(0.0, "Sigma max", gr.Slider, {"minimum": 0.0, "maximum": 100.0, "step": 1.0, "visible": not native}), - "schedulers_sep_compvis": OptionInfo("

CompVis specific config

", "", gr.HTML, {"visible": not native}), + "schedulers_sep_compvis": OptionInfo("

CompVis Config

", "", gr.HTML, {"visible": not native}), 'uni_pc_variant': OptionInfo("bh2", "UniPC variant", gr.Radio, {"choices": ["bh1", "bh2", "vary_coeff"], "visible": not native}), 'uni_pc_skip_type': OptionInfo("time_uniform", "UniPC skip type", gr.Radio, {"choices": ["time_uniform", "time_quadratic", "logSNR"], "visible": not native}), "ddim_discretize": OptionInfo('uniform', "DDIM discretize img2img", gr.Radio, {"choices": ['uniform', 'quad'], "visible": not native}), @@ -849,7 +859,7 @@ def get_default_modes(): "detailer_unload": OptionInfo(False, "Move detailer model to CPU when complete"), "detailer_augment": OptionInfo(True, "Detailer use model augment"), - "postprocessing_sep_face_restore": OptionInfo("

Face restore

", "", gr.HTML), + "postprocessing_sep_face_restore": OptionInfo("

Face Restore

", "", gr.HTML), "face_restoration_model": OptionInfo("None", "Face restoration", gr.Radio, lambda: {"choices": ['None'] + [x.name() for x in face_restorers]}), "code_former_weight": OptionInfo(0.2, "CodeFormer weight parameter", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01}), @@ -879,6 +889,15 @@ def get_default_modes(): "deepbooru_filter_tags": OptionInfo("", "Filter out tags from deepbooru output"), })) +options_templates.update(options_section(('huggingface', "Huggingface"), { + "huggingface_sep": OptionInfo("

Huggingface

", "", gr.HTML), + "diffuser_cache_config": OptionInfo(True, "Use cached model config when available"), + "huggingface_token": OptionInfo('', 'HuggingFace token'), + "diffusers_model_load_variant": OptionInfo("default", "Preferred Model variant", gr.Radio, {"choices": ['default', 'fp32', 'fp16']}), + "diffusers_vae_load_variant": OptionInfo("default", "Preferred VAE variant", gr.Radio, {"choices": ['default', 'fp32', 'fp16']}), + "custom_diffusers_pipeline": OptionInfo('', 'Load custom Diffusers pipeline'), +})) + options_templates.update(options_section(('extra_networks', "Networks"), { "extra_networks_sep1": OptionInfo("

Networks UI

", "", gr.HTML), "extra_networks_show": OptionInfo(True, "UI show on startup"), diff --git a/scripts/cogvideo.py b/scripts/cogvideo.py index c988c05c4..284109857 100644 --- a/scripts/cogvideo.py +++ b/scripts/cogvideo.py @@ -51,7 +51,7 @@ def video_type_change(video_type): with gr.Row(): video_type = gr.Dropdown(label='Video file', choices=['None', 'GIF', 'PNG', 'MP4'], value='None') duration = gr.Slider(label='Duration', minimum=0.25, maximum=30, step=0.25, value=8, visible=False) - with gr.Accordion('Optional init video', open=False): + with gr.Accordion('Optional init image or video', open=False): with gr.Row(): image = gr.Image(value=None, label='Image', type='pil', source='upload', width=256, height=256) video = gr.Video(value=None, label='Video', source='upload', width=256, height=256) @@ -169,25 +169,18 @@ def generate(self, p: processing.StableDiffusionProcessing, model: str): callback_on_step_end=diffusers_callback, callback_on_step_end_tensor_inputs=['latents'], ) - if getattr(p, 'image', False): - if 'I2V' not in model: - shared.log.error(f'CogVideoX: model={model} image input not supported') - return [] - args['image'] = self.image(p, p.image) - args['num_frames'] = p.frames # only txt2vid has num_frames - shared.sd_model = sd_models.switch_pipe(diffusers.CogVideoXImageToVideoPipeline, shared.sd_model) - elif getattr(p, 'video', False): - if 'I2V' in model: - shared.log.error(f'CogVideoX: model={model} image input not supported') - return [] - args['video'] = self.video(p, p.video) - shared.sd_model = sd_models.switch_pipe(diffusers.CogVideoXVideoToVideoPipeline, shared.sd_model) + if 'I2V' in model: + if hasattr(p, 'video') and p.video is not None: + args['video'] = self.video(p, p.video) + shared.sd_model = sd_models.switch_pipe(diffusers.CogVideoXVideoToVideoPipeline, shared.sd_model) + elif (hasattr(p, 'image') and p.image is not None) or (hasattr(p, 'init_images') and len(p.init_images) > 0): + p.init_images = [p.image] if hasattr(p, 'image') and p.image is not None else p.init_images + args['image'] = self.image(p, p.init_images[0]) + shared.sd_model = sd_models.switch_pipe(diffusers.CogVideoXImageToVideoPipeline, shared.sd_model) else: - if 'I2V' in model: - shared.log.error(f'CogVideoX: model={model} image input not supported') - return [] - args['num_frames'] = p.frames # only txt2vid has num_frames shared.sd_model = sd_models.switch_pipe(diffusers.CogVideoXPipeline, shared.sd_model) + args['num_frames'] = p.frames # only txt2vid has num_frames + shared.log.info(f'CogVideoX: class={shared.sd_model.__class__.__name__} frames={p.frames} input={args.get('video', None) or args.get('image', None)}') if debug: shared.log.debug(f'CogVideoX args: {args}') frames = shared.sd_model(**args).frames[0] @@ -199,7 +192,7 @@ def generate(self, p: processing.StableDiffusionProcessing, model: str): errors.display(e, 'CogVideoX') t1 = time.time() its = (len(frames) * p.steps) / (t1 - t0) - shared.log.info(f'CogVideoX: frames={len(frames)} its={its:.2f} time={t1 - t0:.2f}') + shared.log.info(f'CogVideoX: frame={frames[0] if len(frames) > 0 else None} frames={len(frames)} its={its:.2f} time={t1 - t0:.2f}') return frames # auto-executed by the script-callback From f86344501912ad34ac7e68a67f48faad5fa69974 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 10 Dec 2024 10:39:13 -0500 Subject: [PATCH 076/249] warn on quanto with offload Signed-off-by: Vladimir Mandic --- modules/model_quant.py | 7 +++++-- modules/shared.py | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/modules/model_quant.py b/modules/model_quant.py index 9482fe898..03043b33a 100644 --- a/modules/model_quant.py +++ b/modules/model_quant.py @@ -58,11 +58,11 @@ def load_torchao(msg='', silent=False): import torchao ao = torchao fn = f'{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access - log.debug(f'Quantization: type=quanto version={ao.__version__} fn={fn}') # pylint: disable=protected-access + log.debug(f'Quantization: type=torchao version={ao.__version__} fn={fn}') # pylint: disable=protected-access return ao except Exception as e: if len(msg) > 0: - log.error(f"{msg} failed to import optimum.quanto: {e}") + log.error(f"{msg} failed to import torchao: {e}") ao = None if not silent: raise @@ -92,6 +92,7 @@ def load_bnb(msg='', silent=False): def load_quanto(msg='', silent=False): + from modules import shared global quanto # pylint: disable=global-statement if quanto is not None: return quanto @@ -101,6 +102,8 @@ def load_quanto(msg='', silent=False): quanto = optimum_quanto fn = f'{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access log.debug(f'Quantization: type=quanto version={quanto.__version__} fn={fn}') # pylint: disable=protected-access + if shared.opts.diffusers_offload_mode != 'none': + shared.log.error(f'Quantization: type=quanto offload={shared.opts.diffusers_offload_mode} not supported') return quanto except Exception as e: if len(msg) > 0: diff --git a/modules/shared.py b/modules/shared.py index 4dd26d601..bc12c5c15 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -549,7 +549,7 @@ def get_default_modes(): "diffusers_fuse_projections": OptionInfo(False, "Fused projections"), "torch_expandable_segments": OptionInfo(False, "Expandable segments"), "cuda_mem_fraction": OptionInfo(0.0, "Memory limit", gr.Slider, {"minimum": 0, "maximum": 2.0, "step": 0.05}), - "torch_gc_threshold": OptionInfo(80, "GC threshold", gr.Slider, {"minimum": 0, "maximum": 100, "step": 1}), + "torch_gc_threshold": OptionInfo(70, "GC threshold", gr.Slider, {"minimum": 0, "maximum": 100, "step": 1}), "inference_mode": OptionInfo("no-grad", "Inference mode", gr.Radio, {"choices": ["no-grad", "inference-mode", "none"]}), "torch_malloc": OptionInfo("native", "Memory allocator", gr.Radio, {"choices": ['native', 'cudaMallocAsync'] }), @@ -566,7 +566,7 @@ def get_default_modes(): "olive_cache_optimized": OptionInfo(True, 'Olive cache optimized models'), "ipex_sep": OptionInfo("

IPEX

", "", gr.HTML, {"visible": devices.backend == "ipex"}), - "ipex_optimize": OptionInfo([], "IPEX Optimize for Intel GPUs", gr.CheckboxGroup, {"choices": ["Model", "VAE", "Text Encoder", "Upscaler"], "visible": devices.backend == "ipex"}), + "ipex_optimize": OptionInfo([], "IPEX Optimize", gr.CheckboxGroup, {"choices": ["Model", "VAE", "Text Encoder", "Upscaler"], "visible": devices.backend == "ipex"}), "openvino_sep": OptionInfo("

OpenVINO

", "", gr.HTML, {"visible": cmd_opts.use_openvino}), "openvino_devices": OptionInfo([], "OpenVINO devices to use", gr.CheckboxGroup, {"choices": get_openvino_device_list() if cmd_opts.use_openvino else [], "visible": cmd_opts.use_openvino}), # pylint: disable=E0606 From 9bac658b0ce8d8e640175e241e9b97725b22ca15 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 10 Dec 2024 12:34:27 -0500 Subject: [PATCH 077/249] update lora Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 7 ++++--- modules/lora/extra_networks_lora.py | 6 +++++- modules/lora/networks.py | 19 ++++++++++++------- modules/shared.py | 18 ++++++++++-------- wiki | 2 +- 5 files changed, 32 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5412861a8..8ade15e58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,9 +37,8 @@ - LoRA weights are no longer calculated on-the-fly during model execution, but are pre-calculated at the start this results in perceived overhead on generate startup, but results in overall faster execution as LoRA does not need to be processed on each step thanks @AI-Casanova - - *note*: LoRA weights backups are required so LoRA can be unapplied, but can take quite a lot of system memory - if you know you will not need to unapply LoRA, you can disable backups in *settings -> networks -> lora fuse* - in which case, you need to reload model to unapply LoRA + - LoRA weights can be applied/unapplied as on each generate or they can store weights backups for later use + this setting has large performance and resource implications, see [Offload](https://github.com/vladmandic/automatic/wiki/Offload) wiki for details - **Model loader** improvements: - detect model components on model load fail - allow passing absolute path to model loader @@ -98,6 +97,8 @@ - fix prompt caching - fix xyz grid skip final pass - fix sd upscale script +- fix cogvideox-i2v +- lora auto-apply tags remove duplicates ## Update for 2024-11-21 diff --git a/modules/lora/extra_networks_lora.py b/modules/lora/extra_networks_lora.py index 135df1ccb..42c4a92f6 100644 --- a/modules/lora/extra_networks_lora.py +++ b/modules/lora/extra_networks_lora.py @@ -44,6 +44,8 @@ def prompt(p): loaded.tags = loaded.tags[:shared.opts.lora_apply_tags] all_tags.extend(loaded.tags) if len(all_tags) > 0: + all_tags = list(set(all_tags)) + all_tags = [t for t in all_tags if t not in p.prompt] shared.log.debug(f"Load network: type=LoRA tags={all_tags} max={shared.opts.lora_apply_tags} apply") all_tags = ', '.join(all_tags) p.extra_generation_params["LoRA tags"] = all_tags @@ -121,13 +123,15 @@ def activate(self, p, params_list, step=0, include=[], exclude=[]): # shared.log.debug(f'Activate network: type=LoRA model="{shared.opts.sd_model_checkpoint}"') self.active = True self.model = shared.opts.sd_model_checkpoint + if 'text_encoder' in include: + networks.timer.clear(complete=True) names, te_multipliers, unet_multipliers, dyn_dims = parse(p, params_list, step) networks.network_load(names, te_multipliers, unet_multipliers, dyn_dims) # load networks.network_activate(include, exclude) if len(networks.loaded_networks) > 0 and len(networks.applied_layers) > 0 and step == 0: infotext(p) prompt(p) - shared.log.info(f'Load network: type=LoRA apply={[n.name for n in networks.loaded_networks]} te={te_multipliers} unet={unet_multipliers} time={networks.timer.summary}') + shared.log.info(f'Load network: type=LoRA apply={[n.name for n in networks.loaded_networks]} mode={"fuse" if shared.opts.lora_fuse_diffusers else "backup"} te={te_multipliers} unet={unet_multipliers} time={networks.timer.summary}') def deactivate(self, p): if shared.native and len(networks.diffuser_loaded) > 0: diff --git a/modules/lora/networks.py b/modules/lora/networks.py index edd82f3e4..ada6f833d 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -42,6 +42,7 @@ # section: load networks from disk def load_diffusers(name, network_on_disk, lora_scale=shared.opts.extra_networks_default_multiplier) -> Union[network.Network, None]: + t0 = time.time() name = name.replace(".", "_") shared.log.debug(f'Load network: type=LoRA name="{name}" file="{network_on_disk.filename}" detected={network_on_disk.sd_version} method=diffusers scale={lora_scale} fuse={shared.opts.lora_fuse_diffusers}') if not shared.native: @@ -67,6 +68,7 @@ def load_diffusers(name, network_on_disk, lora_scale=shared.opts.extra_networks_ diffuser_scales.append(lora_scale) net = network.Network(name, network_on_disk) net.mtime = os.path.getmtime(network_on_disk.filename) + timer.activate += time.time() - t0 return net @@ -256,10 +258,12 @@ def network_load(names, te_multipliers=None, unet_multipliers=None, dyn_dims=Non if len(diffuser_loaded) > 0: shared.log.debug(f'Load network: type=LoRA loaded={diffuser_loaded} available={shared.sd_model.get_list_adapters()} active={shared.sd_model.get_active_adapters()} scales={diffuser_scales}') try: + t0 = time.time() shared.sd_model.set_adapters(adapter_names=diffuser_loaded, adapter_weights=diffuser_scales) if shared.opts.lora_fuse_diffusers: shared.sd_model.fuse_lora(adapter_names=diffuser_loaded, lora_scale=1.0, fuse_unet=True, fuse_text_encoder=True) # fuse uses fixed scale since later apply does the scaling shared.sd_model.unload_lora_weights() + timer.activate += time.time() - t0 except Exception as e: shared.log.error(f'Load network: type=LoRA {e}') if debug: @@ -301,16 +305,15 @@ def network_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.n bnb = model_quant.load_bnb('Load network: type=LoRA', silent=True) if bnb is not None: with devices.inference_context(): - weights_backup = bnb.functional.dequantize_4bit(weight, quant_state=weight.quant_state, quant_type=weight.quant_type, blocksize=weight.blocksize,) + self.network_weights_backup = bnb.functional.dequantize_4bit(weight, quant_state=weight.quant_state, quant_type=weight.quant_type, blocksize=weight.blocksize,) self.quant_state = weight.quant_state self.quant_type = weight.quant_type self.blocksize = weight.blocksize else: weights_backup = weight.clone() - weights_backup = weights_backup.to(devices.cpu) + self.network_weights_backup = weights_backup.to(devices.cpu) else: - weights_backup = weight.clone() - weights_backup = weights_backup.to(devices.cpu) + self.network_weights_backup = weight.clone().to(devices.cpu) bias_backup = getattr(self, "network_bias_backup", None) if bias_backup is None: @@ -331,7 +334,10 @@ def network_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.n def network_calc_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv], network_layer_name: str): if shared.opts.diffusers_offload_mode == "none": - self.to(devices.device) + try: + self.to(devices.device) + except Exception: + pass batch_updown = None batch_ex_bias = None for net in loaded_networks: @@ -501,7 +507,6 @@ def network_deactivate(): def network_activate(include=[], exclude=[]): t0 = time.time() - timer.clear(complete=True) sd_model = getattr(shared.sd_model, "pipe", shared.sd_model) # wrapped model compatiblility if shared.opts.diffusers_offload_mode == "sequential": sd_models.disable_offload(sd_model) @@ -552,7 +557,7 @@ def network_activate(include=[], exclude=[]): if task is not None and len(applied_layers) == 0: pbar.remove_task(task) # hide progress bar for no action weights_devices, weights_dtypes = list(set([x for x in weights_devices if x is not None])), list(set([x for x in weights_dtypes if x is not None])) # noqa: C403 # pylint: disable=R1718 - timer.activate = time.time() - t0 + timer.activate += time.time() - t0 if debug and len(loaded_networks) > 0: shared.log.debug(f'Load network: type=LoRA networks={len(loaded_networks)} components={components} modules={total} apply={len(applied_layers)} device={weights_devices} dtype={weights_dtypes} backup={backup_size} fuse={shared.opts.lora_fuse_diffusers} time={timer.summary}') modules.clear() diff --git a/modules/shared.py b/modules/shared.py index bc12c5c15..21d3004ca 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -914,22 +914,24 @@ def get_default_modes(): "extra_networks_model_sep": OptionInfo("

Models

", "", gr.HTML), "extra_network_reference": OptionInfo(False, "Use reference values when available", gr.Checkbox), - "extra_networks_embed_sep": OptionInfo("

Embeddings

", "", gr.HTML), - "diffusers_convert_embed": OptionInfo(False, "Auto-convert SD 1.5 embeddings to SDXL ", gr.Checkbox, {"visible": native}), - "extra_networks_styles_sep": OptionInfo("

Styles

", "", gr.HTML), - "extra_networks_styles": OptionInfo(True, "Show built-in styles"), - "extra_networks_wildcard_sep": OptionInfo("

Wildcards

", "", gr.HTML), - "wildcards_enabled": OptionInfo(True, "Enable file wildcards support"), + "extra_networks_lora_sep": OptionInfo("

LoRA

", "", gr.HTML), "extra_networks_default_multiplier": OptionInfo(1.0, "Default strength", gr.Slider, {"minimum": 0.0, "maximum": 2.0, "step": 0.01}), "lora_preferred_name": OptionInfo("filename", "LoRA preferred name", gr.Radio, {"choices": ["filename", "alias"], "visible": False}), - "lora_add_hashes_to_infotext": OptionInfo(False, "LoRA add hash info"), + "lora_add_hashes_to_infotext": OptionInfo(False, "LoRA add hash info to metadata"), "lora_fuse_diffusers": OptionInfo(True, "LoRA fuse directly to model"), "lora_force_diffusers": OptionInfo(False if not cmd_opts.use_openvino else True, "LoRA force loading of all models using Diffusers"), "lora_maybe_diffusers": OptionInfo(False, "LoRA force loading of specific models using Diffusers"), "lora_apply_tags": OptionInfo(0, "LoRA auto-apply tags", gr.Slider, {"minimum": -1, "maximum": 32, "step": 1}), "lora_in_memory_limit": OptionInfo(0, "LoRA memory cache", gr.Slider, {"minimum": 0, "maximum": 24, "step": 1}), - "lora_quant": OptionInfo("NF4","LoRA precision in quantized models", gr.Radio, {"choices": ["NF4", "FP4"]}), + "lora_quant": OptionInfo("NF4","LoRA precision when quantized", gr.Radio, {"choices": ["NF4", "FP4"]}), + + "extra_networks_styles_sep": OptionInfo("

Styles

", "", gr.HTML), + "extra_networks_styles": OptionInfo(True, "Show built-in styles"), + "extra_networks_embed_sep": OptionInfo("

Embeddings

", "", gr.HTML), + "diffusers_convert_embed": OptionInfo(False, "Auto-convert SD15 embeddings to SDXL ", gr.Checkbox, {"visible": native}), + "extra_networks_wildcard_sep": OptionInfo("

Wildcards

", "", gr.HTML), + "wildcards_enabled": OptionInfo(True, "Enable file wildcards support"), })) options_templates.update(options_section((None, "Internal options"), { diff --git a/wiki b/wiki index 8960da514..95f174900 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 8960da514e9aff4a5d47402925c9498536443379 +Subproject commit 95f1749005d56be490dab95cf92f4ca576d10396 From 72d946b2855c46f870034beaf6b747e2ab0e5de3 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 10 Dec 2024 12:38:19 -0500 Subject: [PATCH 078/249] update bug report Signed-off-by: Vladimir Mandic --- .github/ISSUE_TEMPLATE/bug_report.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index cf176d6cf..a40320c63 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -106,10 +106,14 @@ body: - StableDiffusion 1.5 - StableDiffusion 2.1 - StableDiffusion XL - - StableDiffusion 3 - - PixArt + - StableDiffusion 3.x - StableCascade + - FLUX.1 + - PixArt - Kandinsky + - Playground + - AuraFlow + - Any Video Model - Other default: 0 validations: From ee4b63381f8a138e4c73696e6859d281ff85242a Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 10 Dec 2024 15:49:20 -0500 Subject: [PATCH 079/249] optimize balanced offload Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 6 ++- modules/memstats.py | 8 +++- modules/processing_diffusers.py | 4 +- modules/processing_vae.py | 6 +-- modules/sd_models.py | 66 ++++++++++++++++++--------------- modules/shared.py | 1 + scripts/cogvideo.py | 2 +- 7 files changed, 54 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ade15e58..4f6b10909 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,8 +50,10 @@ - **Memory** improvements: - faster and more compatible *balanced offload* mode - balanced offload: units are now in percentage instead of bytes - - balanced offload: add both high and low watermark - default is 25% for low-watermark (skip offload if memory usage is below 25%) and 70% high-watermark (must offload if memory usage is above 70%) + - balanced offload: add both high and low watermark and pinned threshold, defaults as below + 25% for low-watermark: skip offload if memory usage is below 25% + 70% high-watermark: must offload if memory usage is above 70% + 15% pin-watermark: any model component smaller than 15% of total memory is pinned and not offloaded - change-in-behavior: low-end systems, triggered by either `lowvrwam` or by detection of <=4GB will use *sequential offload* all other systems use *balanced offload* by default (can be changed in settings) diff --git a/modules/memstats.py b/modules/memstats.py index fd5f152a0..d43e5bbfa 100644 --- a/modules/memstats.py +++ b/modules/memstats.py @@ -4,6 +4,8 @@ from modules import shared, errors fail_once = False +mem = {} + def gb(val: float): return round(val / 1024 / 1024 / 1024, 2) @@ -11,7 +13,7 @@ def gb(val: float): def memory_stats(): global fail_once # pylint: disable=global-statement - mem = {} + mem.clear() try: process = psutil.Process(os.getpid()) res = process.memory_info() @@ -41,6 +43,10 @@ def memory_stats(): return mem +def memory_cache(): + return mem + + def ram_stats(): try: process = psutil.Process(os.getpid()) diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 627eb281f..9f12e44ab 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -349,7 +349,7 @@ def process_refine(p: processing.StableDiffusionProcessing, output): def process_decode(p: processing.StableDiffusionProcessing, output): - shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model, exclude=['vae']) if output is not None: if not hasattr(output, 'images') and hasattr(output, 'frames'): shared.log.debug(f'Generated: frames={len(output.frames[0])}') @@ -463,7 +463,7 @@ def process_diffusers(p: processing.StableDiffusionProcessing): timer.process.record('decode') shared.sd_model = orig_pipeline - shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) + # shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) if p.state == '': global last_p # pylint: disable=global-statement diff --git a/modules/processing_vae.py b/modules/processing_vae.py index 1c4a45f07..77a89c512 100644 --- a/modules/processing_vae.py +++ b/modules/processing_vae.py @@ -104,8 +104,6 @@ def full_vae_decode(latents, model): if shared.opts.diffusers_move_unet and not getattr(model, 'has_accelerate', False): base_device = sd_models.move_base(model, devices.cpu) - if shared.opts.diffusers_offload_mode == "balanced": - shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) elif shared.opts.diffusers_offload_mode != "sequential": sd_models.move_model(model.vae, devices.device) @@ -159,8 +157,8 @@ def full_vae_decode(latents, model): model.vae.apply(sd_models.convert_to_faketensors) devices.torch_gc(force=True) - if shared.opts.diffusers_offload_mode == "balanced": - shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) + # if shared.opts.diffusers_offload_mode == "balanced": + # shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) elif shared.opts.diffusers_move_unet and not getattr(model, 'has_accelerate', False) and base_device is not None: sd_models.move_base(model, base_device) t1 = time.time() diff --git a/modules/sd_models.py b/modules/sd_models.py index 8853916e4..bd69ba45b 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -18,7 +18,7 @@ from ldm.util import instantiate_from_config from modules import paths, shared, shared_state, modelloader, devices, script_callbacks, sd_vae, sd_unet, errors, sd_models_config, sd_models_compile, sd_hijack_accelerate, sd_detect from modules.timer import Timer, process as process_timer -from modules.memstats import memory_stats +from modules.memstats import memory_stats, memory_cache from modules.modeldata import model_data from modules.sd_checkpoint import CheckpointInfo, select_checkpoint, list_models, checkpoints_list, checkpoint_titles, get_closet_checkpoint_match, model_hash, update_model_hashes, setup_model, write_metadata, read_metadata_from_safetensors # pylint: disable=unused-import @@ -416,9 +416,10 @@ def detach_hook(self, module): offload_hook_instance = None +offload_component_map = {} -def apply_balanced_offload(sd_model): +def apply_balanced_offload(sd_model, exclude=[]): global offload_hook_instance # pylint: disable=global-statement if shared.opts.diffusers_offload_mode != "balanced": return sd_model @@ -428,8 +429,6 @@ def apply_balanced_offload(sd_model): excluded = ['OmniGenPipeline'] if sd_model.__class__.__name__ in excluded: return sd_model - fn = f'{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access - debug_move(f'Apply offload: type=balanced fn={fn}') checkpoint_name = sd_model.sd_checkpoint_info.name if getattr(sd_model, "sd_checkpoint_info", None) is not None else None if checkpoint_name is None: checkpoint_name = sd_model.__class__.__name__ @@ -442,32 +441,38 @@ def apply_balanced_offload_to_module(pipe): keys = pipe._internal_dict.keys() # pylint: disable=protected-access else: keys = get_signature(pipe).keys() + keys = [k for k in keys if k not in exclude and not k.startswith('_')] for module_name in keys: # pylint: disable=protected-access module = getattr(pipe, module_name, None) - if isinstance(module, torch.nn.Module): - network_layer_name = getattr(module, "network_layer_name", None) - device_map = getattr(module, "balanced_offload_device_map", None) - max_memory = getattr(module, "balanced_offload_max_memory", None) - module = accelerate.hooks.remove_hook_from_module(module, recurse=True) - try: - do_offload = used_gpu > 100 * shared.opts.diffusers_offload_min_gpu_memory - debug_move(f'Balanced offload: gpu={used_gpu} ram={used_ram} current={module.device} dtype={module.dtype} op={"move" if do_offload else "skip"} component={module.__class__.__name__}') - if do_offload: - module = module.to(devices.cpu) - used_gpu, used_ram = devices.torch_gc(fast=True, force=True) - except Exception as e: - if 'bitsandbytes' not in str(e): - shared.log.error(f'Balanced offload: module={module_name} {e}') - if os.environ.get('SD_MOVE_DEBUG', None): - errors.display(e, f'Balanced offload: module={module_name}') - module.offload_dir = os.path.join(shared.opts.accelerate_offload_path, checkpoint_name, module_name) - module = accelerate.hooks.add_hook_to_module(module, offload_hook_instance, append=True) - module._hf_hook.execution_device = torch.device(devices.device) # pylint: disable=protected-access - if network_layer_name: - module.network_layer_name = network_layer_name - if device_map and max_memory: - module.balanced_offload_device_map = device_map - module.balanced_offload_max_memory = max_memory + if not isinstance(module, torch.nn.Module): + continue + network_layer_name = getattr(module, "network_layer_name", None) + device_map = getattr(module, "balanced_offload_device_map", None) + max_memory = getattr(module, "balanced_offload_max_memory", None) + module = accelerate.hooks.remove_hook_from_module(module, recurse=True) + module_size = offload_component_map.get(module_name, None) + if module_size is None: + module_size = sum(p.numel()*p.element_size() for p in module.parameters(recurse=True)) / 1024 / 1024 / 1024 + offload_component_map[module_name] = module_size + do_offload = (used_gpu > 100 * shared.opts.diffusers_offload_min_gpu_memory) and (module_size > shared.gpu_memory * shared.opts.diffusers_offload_pin_gpu_memory) + try: + debug_move(f'Balanced offload: gpu={used_gpu} ram={used_ram} current={module.device} dtype={module.dtype} op={"move" if do_offload else "skip"} component={module.__class__.__name__} size={module_size:.3f}') + if do_offload and module.device != devices.cpu: + module = module.to(devices.cpu) + used_gpu, used_ram = devices.torch_gc(fast=True, force=True) + except Exception as e: + if 'bitsandbytes' not in str(e): + shared.log.error(f'Balanced offload: module={module_name} {e}') + if os.environ.get('SD_MOVE_DEBUG', None): + errors.display(e, f'Balanced offload: module={module_name}') + module.offload_dir = os.path.join(shared.opts.accelerate_offload_path, checkpoint_name, module_name) + module = accelerate.hooks.add_hook_to_module(module, offload_hook_instance, append=True) + module._hf_hook.execution_device = torch.device(devices.device) # pylint: disable=protected-access + if network_layer_name: + module.network_layer_name = network_layer_name + if device_map and max_memory: + module.balanced_offload_device_map = device_map + module.balanced_offload_max_memory = max_memory apply_balanced_offload_to_module(sd_model) if hasattr(sd_model, "pipe"): @@ -478,7 +483,10 @@ def apply_balanced_offload_to_module(pipe): apply_balanced_offload_to_module(sd_model.decoder_pipe) set_accelerate(sd_model) devices.torch_gc(fast=True) - process_timer.add('offload', time.time() - t0) + t = time.time() - t0 + process_timer.add('offload', t) + fn = f'{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access + debug_move(f'Apply offload: time={t:.2f} type=balanced fn={fn}') return sd_model diff --git a/modules/shared.py b/modules/shared.py index 21d3004ca..41e703285 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -483,6 +483,7 @@ def get_default_modes(): "diffusers_offload_mode": OptionInfo(startup_offload_mode, "Model offload mode", gr.Radio, {"choices": ['none', 'balanced', 'model', 'sequential']}), "diffusers_offload_min_gpu_memory": OptionInfo(0.25, "Balanced offload GPU low watermark", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01 }), "diffusers_offload_max_gpu_memory": OptionInfo(0.70, "Balanced offload GPU high watermark", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01 }), + "diffusers_offload_pin_gpu_memory": OptionInfo(0.15, "Balanced offload GPU pin watermark", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01 }), "diffusers_offload_max_cpu_memory": OptionInfo(0.90, "Balanced offload CPU high watermark", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01 }), "advanced_sep": OptionInfo("

Advanced Options

", "", gr.HTML), diff --git a/scripts/cogvideo.py b/scripts/cogvideo.py index 284109857..a5efcd3e6 100644 --- a/scripts/cogvideo.py +++ b/scripts/cogvideo.py @@ -180,7 +180,7 @@ def generate(self, p: processing.StableDiffusionProcessing, model: str): else: shared.sd_model = sd_models.switch_pipe(diffusers.CogVideoXPipeline, shared.sd_model) args['num_frames'] = p.frames # only txt2vid has num_frames - shared.log.info(f'CogVideoX: class={shared.sd_model.__class__.__name__} frames={p.frames} input={args.get('video', None) or args.get('image', None)}') + shared.log.info(f"CogVideoX: class={shared.sd_model.__class__.__name__} frames={p.frames} input={args.get('video', None) or args.get('image', None)}") if debug: shared.log.debug(f'CogVideoX args: {args}') frames = shared.sd_model(**args).frames[0] From bb80f2dad7436f4ea001e9cb5e7d8f40306197b3 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 11 Dec 2024 12:06:03 -0500 Subject: [PATCH 080/249] update balanced offload Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 7 +++--- modules/devices.py | 54 ++++++++++++++++++++++++-------------------- modules/sd_models.py | 52 ++++++++++++++++++++++++++++-------------- modules/shared.py | 3 +-- wiki | 2 +- 5 files changed, 70 insertions(+), 48 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f6b10909..4b8f4994c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,10 +50,9 @@ - **Memory** improvements: - faster and more compatible *balanced offload* mode - balanced offload: units are now in percentage instead of bytes - - balanced offload: add both high and low watermark and pinned threshold, defaults as below - 25% for low-watermark: skip offload if memory usage is below 25% - 70% high-watermark: must offload if memory usage is above 70% - 15% pin-watermark: any model component smaller than 15% of total memory is pinned and not offloaded + - balanced offload: add both high and low watermark, defaults as below + `0.25` for low-watermark: skip offload if memory usage is below 25% + `0.70` high-watermark: must offload if memory usage is above 70% - change-in-behavior: low-end systems, triggered by either `lowvrwam` or by detection of <=4GB will use *sequential offload* all other systems use *balanced offload* by default (can be changed in settings) diff --git a/modules/devices.py b/modules/devices.py index 71eef5726..51b770481 100644 --- a/modules/devices.py +++ b/modules/devices.py @@ -187,27 +187,34 @@ def get_device_for(task): # pylint: disable=unused-argument def torch_gc(force=False, fast=False): + def get_stats(): + mem_dict = memstats.memory_stats() + gpu_dict = mem_dict.get('gpu', {}) + ram_dict = mem_dict.get('ram', {}) + oom = gpu_dict.get('oom', 0) + ram = ram_dict.get('used', 0) + if backend == "directml": + gpu = torch.cuda.memory_allocated() / (1 << 30) + else: + gpu = gpu_dict.get('used', 0) + used_gpu = round(100 * gpu / gpu_dict.get('total', 1)) if gpu_dict.get('total', 1) > 1 else 0 + used_ram = round(100 * ram / ram_dict.get('total', 1)) if ram_dict.get('total', 1) > 1 else 0 + return gpu, used_gpu, ram, used_ram, oom + + global previous_oom # pylint: disable=global-statement import gc from modules import timer, memstats from modules.shared import cmd_opts + t0 = time.time() - mem = memstats.memory_stats() - gpu = mem.get('gpu', {}) - ram = mem.get('ram', {}) - oom = gpu.get('oom', 0) - if backend == "directml": - used_gpu = round(100 * torch.cuda.memory_allocated() / (1 << 30) / gpu.get('total', 1)) if gpu.get('total', 1) > 1 else 0 - else: - used_gpu = round(100 * gpu.get('used', 0) / gpu.get('total', 1)) if gpu.get('total', 1) > 1 else 0 - used_ram = round(100 * ram.get('used', 0) / ram.get('total', 1)) if ram.get('total', 1) > 1 else 0 - global previous_oom # pylint: disable=global-statement + gpu, used_gpu, ram, used_ram, oom = get_stats() threshold = 0 if (cmd_opts.lowvram and not cmd_opts.use_zluda) else opts.torch_gc_threshold collected = 0 if force or threshold == 0 or used_gpu >= threshold or used_ram >= threshold: force = True if oom > previous_oom: previous_oom = oom - log.warning(f'Torch GPU out-of-memory error: {mem}') + log.warning(f'Torch GPU out-of-memory error: {memstats.memory_stats()}') force = True if force: # actual gc @@ -215,25 +222,24 @@ def torch_gc(force=False, fast=False): if cuda_ok: try: with torch.cuda.device(get_cuda_device_string()): + torch.cuda.synchronize() torch.cuda.empty_cache() # cuda gc torch.cuda.ipc_collect() except Exception: pass + else: + return gpu, ram t1 = time.time() - if 'gc' not in timer.process.records: - timer.process.records['gc'] = 0 - timer.process.records['gc'] += t1 - t0 - if not force or collected == 0: - return used_gpu, used_ram - mem = memstats.memory_stats() - saved = round(gpu.get('used', 0) - mem.get('gpu', {}).get('used', 0), 2) - before = { 'gpu': gpu.get('used', 0), 'ram': ram.get('used', 0) } - after = { 'gpu': mem.get('gpu', {}).get('used', 0), 'ram': mem.get('ram', {}).get('used', 0), 'retries': mem.get('retries', 0), 'oom': mem.get('oom', 0) } - utilization = { 'gpu': used_gpu, 'ram': used_ram, 'threshold': threshold } - results = { 'collected': collected, 'saved': saved } + timer.process.add('gc', t1 - t0) + + new_gpu, new_used_gpu, new_ram, new_used_ram, oom = get_stats() + before = { 'gpu': gpu, 'ram': ram } + after = { 'gpu': new_gpu, 'ram': new_ram, 'oom': oom } + utilization = { 'gpu': new_used_gpu, 'ram': new_used_ram, 'threshold': threshold } + results = { 'saved': round(gpu - new_gpu, 2), 'collected': collected } fn = f'{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access - log.debug(f'GC: utilization={utilization} gc={results} before={before} after={after} device={torch.device(get_optimal_device_name())} fn={fn} time={round(t1 - t0, 2)}') # pylint: disable=protected-access - return used_gpu, used_ram + log.debug(f'GC: utilization={utilization} gc={results} before={before} after={after} device={torch.device(get_optimal_device_name())} fn={fn} time={round(t1 - t0, 2)}') + return new_gpu, new_ram def set_cuda_sync_mode(mode): diff --git a/modules/sd_models.py b/modules/sd_models.py index bd69ba45b..85a5dc5d7 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -18,7 +18,7 @@ from ldm.util import instantiate_from_config from modules import paths, shared, shared_state, modelloader, devices, script_callbacks, sd_vae, sd_unet, errors, sd_models_config, sd_models_compile, sd_hijack_accelerate, sd_detect from modules.timer import Timer, process as process_timer -from modules.memstats import memory_stats, memory_cache +from modules.memstats import memory_stats from modules.modeldata import model_data from modules.sd_checkpoint import CheckpointInfo, select_checkpoint, list_models, checkpoints_list, checkpoint_titles, get_closet_checkpoint_match, model_hash, update_model_hashes, setup_model, write_metadata, read_metadata_from_safetensors # pylint: disable=unused-import @@ -35,6 +35,8 @@ diffusers_version = int(diffusers.__version__.split('.')[1]) checkpoint_tiles = checkpoint_titles # legacy compatibility should_offload = ['sc', 'sd3', 'f1', 'hunyuandit', 'auraflow', 'omnigen'] +offload_hook_instance = None +offload_component_map = {} class NoWatermark: @@ -415,10 +417,6 @@ def detach_hook(self, module): return module -offload_hook_instance = None -offload_component_map = {} - - def apply_balanced_offload(sd_model, exclude=[]): global offload_hook_instance # pylint: disable=global-statement if shared.opts.diffusers_offload_mode != "balanced": @@ -433,6 +431,29 @@ def apply_balanced_offload(sd_model, exclude=[]): if checkpoint_name is None: checkpoint_name = sd_model.__class__.__name__ + def get_pipe_modules(pipe): + if hasattr(pipe, "_internal_dict"): + modules_names = pipe._internal_dict.keys() # pylint: disable=protected-access + else: + modules_names = get_signature(pipe).keys() + modules_names = [m for m in modules_names if m not in exclude and not m.startswith('_')] + modules = {} + for module_name in modules_names: + module_size = offload_component_map.get(module_name, None) + if module_size is None: + module = getattr(pipe, module_name, None) + if not isinstance(module, torch.nn.Module): + continue + try: + module_size = sum(p.numel()*p.element_size() for p in module.parameters(recurse=True)) / 1024 / 1024 / 1024 + except Exception as e: + shared.log.error(f'Balanced offload: module={module_name} {e}') + module_size = 0 + offload_component_map[module_name] = module_size + modules[module_name] = module_size + modules = sorted(modules.items(), key=lambda x: x[1], reverse=True) + return modules + def apply_balanced_offload_to_module(pipe): used_gpu, used_ram = devices.torch_gc(fast=True) if hasattr(pipe, "pipe"): @@ -442,24 +463,20 @@ def apply_balanced_offload_to_module(pipe): else: keys = get_signature(pipe).keys() keys = [k for k in keys if k not in exclude and not k.startswith('_')] - for module_name in keys: # pylint: disable=protected-access + for module_name, module_size in get_pipe_modules(pipe): # pylint: disable=protected-access module = getattr(pipe, module_name, None) - if not isinstance(module, torch.nn.Module): - continue network_layer_name = getattr(module, "network_layer_name", None) device_map = getattr(module, "balanced_offload_device_map", None) max_memory = getattr(module, "balanced_offload_max_memory", None) module = accelerate.hooks.remove_hook_from_module(module, recurse=True) - module_size = offload_component_map.get(module_name, None) - if module_size is None: - module_size = sum(p.numel()*p.element_size() for p in module.parameters(recurse=True)) / 1024 / 1024 / 1024 - offload_component_map[module_name] = module_size - do_offload = (used_gpu > 100 * shared.opts.diffusers_offload_min_gpu_memory) and (module_size > shared.gpu_memory * shared.opts.diffusers_offload_pin_gpu_memory) + perc_gpu = used_gpu / shared.gpu_memory try: - debug_move(f'Balanced offload: gpu={used_gpu} ram={used_ram} current={module.device} dtype={module.dtype} op={"move" if do_offload else "skip"} component={module.__class__.__name__} size={module_size:.3f}') - if do_offload and module.device != devices.cpu: - module = module.to(devices.cpu) - used_gpu, used_ram = devices.torch_gc(fast=True, force=True) + prev_gpu = used_gpu + do_offload = (perc_gpu > shared.opts.diffusers_offload_min_gpu_memory) and (module.device != devices.cpu) + if do_offload: + module = module.to(devices.cpu, non_blocking=True) + used_gpu -= module_size + debug_move(f'Balanced offload: op={"move" if do_offload else "skip"} gpu={prev_gpu:.3f}:{used_gpu:.3f} perc={perc_gpu:.2f} ram={used_ram:.3f} current={module.device} dtype={module.dtype} component={module.__class__.__name__} size={module_size:.3f}') except Exception as e: if 'bitsandbytes' not in str(e): shared.log.error(f'Balanced offload: module={module_name} {e}') @@ -473,6 +490,7 @@ def apply_balanced_offload_to_module(pipe): if device_map and max_memory: module.balanced_offload_device_map = device_map module.balanced_offload_max_memory = max_memory + devices.torch_gc(fast=True, force=True) apply_balanced_offload_to_module(sd_model) if hasattr(sd_model, "pipe"): diff --git a/modules/shared.py b/modules/shared.py index 41e703285..08a615550 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -483,8 +483,7 @@ def get_default_modes(): "diffusers_offload_mode": OptionInfo(startup_offload_mode, "Model offload mode", gr.Radio, {"choices": ['none', 'balanced', 'model', 'sequential']}), "diffusers_offload_min_gpu_memory": OptionInfo(0.25, "Balanced offload GPU low watermark", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01 }), "diffusers_offload_max_gpu_memory": OptionInfo(0.70, "Balanced offload GPU high watermark", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01 }), - "diffusers_offload_pin_gpu_memory": OptionInfo(0.15, "Balanced offload GPU pin watermark", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01 }), - "diffusers_offload_max_cpu_memory": OptionInfo(0.90, "Balanced offload CPU high watermark", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01 }), + "diffusers_offload_max_cpu_memory": OptionInfo(0.90, "Balanced offload CPU high watermark", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01, "visible": False }), "advanced_sep": OptionInfo("

Advanced Options

", "", gr.HTML), "sd_checkpoint_autoload": OptionInfo(True, "Model autoload on start"), diff --git a/wiki b/wiki index 95f174900..db828893c 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 95f1749005d56be490dab95cf92f4ca576d10396 +Subproject commit db828893c803f1d5d0180cfe09689884bf27af2d From bbdf5fc53e5019370109791584e415dadea181e0 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 11 Dec 2024 12:32:34 -0500 Subject: [PATCH 081/249] add SD_NO_CACHE env variable Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 1 + modules/files_cache.py | 8 ++++---- modules/shared.py | 4 +++- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b8f4994c..f215b84ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -83,6 +83,7 @@ ### Fixes +- add `SD_NO_CACHE=true` env variable to disable file/folder caching - update `diffusers` - fix README links - fix sdxl controlnet single-file loader diff --git a/modules/files_cache.py b/modules/files_cache.py index fa2241afc..d65e0f4f4 100644 --- a/modules/files_cache.py +++ b/modules/files_cache.py @@ -6,6 +6,7 @@ from installer import log +do_cache_folders = os.environ.get('SD_NO_CACHE', None) is None class Directory: # forward declaration ... @@ -87,8 +88,6 @@ def is_stale(self) -> bool: return not self.is_directory or self.mtime != self.live_mtime - - class DirectoryCache(UserDict, DirectoryCollection): def __delattr__(self, directory_path: str) -> None: directory: Directory = get_directory(directory_path, fetch=False) @@ -126,7 +125,7 @@ def clean_directory(directory: Directory, /, recursive: RecursiveType=False) -> return is_clean -def get_directory(directory_or_path: str, /, fetch:bool=True) -> Union[Directory, None]: +def get_directory(directory_or_path: str, /, fetch: bool=True) -> Union[Directory, None]: if isinstance(directory_or_path, Directory): if directory_or_path.is_directory: return directory_or_path @@ -136,8 +135,9 @@ def get_directory(directory_or_path: str, /, fetch:bool=True) -> Union[Directory if not cache_folders.get(directory_or_path, None): if fetch: directory = fetch_directory(directory_path=directory_or_path) - if directory: + if directory and do_cache_folders: cache_folders[directory_or_path] = directory + return directory else: clean_directory(cache_folders[directory_or_path]) return cache_folders[directory_or_path] if directory_or_path in cache_folders else None diff --git a/modules/shared.py b/modules/shared.py index 08a615550..ffba92442 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -16,7 +16,7 @@ import orjson import diffusers from rich.console import Console -from modules import errors, devices, shared_items, shared_state, cmd_args, theme, history +from modules import errors, devices, shared_items, shared_state, cmd_args, theme, history, files_cache from modules.paths import models_path, script_path, data_path, sd_configs_path, sd_default_config, sd_model_file, default_sd_model_file, extensions_dir, extensions_builtin_dir # pylint: disable=W0611 from modules.dml import memory_providers, default_memory_provider, directml_do_hijack from modules.onnx_impl import initialize_onnx, execution_providers @@ -238,6 +238,8 @@ def default(obj): mem_stat = memory_stats() gpu_memory = mem_stat['gpu']['total'] if "gpu" in mem_stat else 0 native = backend == Backend.DIFFUSERS +if not files_cache.do_cache_folders: + log.warning('File cache disabled: ') class OptionInfo: From 1b8f823c8e86680cd588ea8c57d79ee94cde5403 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 11 Dec 2024 13:10:05 -0500 Subject: [PATCH 082/249] lora add erorr handler for partial offload Signed-off-by: Vladimir Mandic --- modules/lora/networks.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/modules/lora/networks.py b/modules/lora/networks.py index ada6f833d..5d285af95 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -346,7 +346,10 @@ def network_calc_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn. continue try: t0 = time.time() - weight = self.weight.to(devices.device) + try: + weight = self.weight.to(devices.device) + except Exception: + weight = self.weight updown, ex_bias = module.calc_updown(weight) if batch_updown is not None and updown is not None: batch_updown += updown.to(batch_updown.device) @@ -389,7 +392,10 @@ def network_apply_direct(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn. if updown is not None: if deactivate: updown *= -1 - new_weight = self.weight.to(devices.device) + updown.to(devices.device) + try: + new_weight = self.weight.to(devices.device) + updown.to(devices.device) + except Exception: + new_weight = self.weight + updown if getattr(self, "quant_type", None) in ['nf4', 'fp4'] and bnb is not None: self.weight = bnb.nn.Params4bit(new_weight, quant_state=self.quant_state, quant_type=self.quant_type, blocksize=self.blocksize) else: From 7b4696301175e70fd31d530f7314bed07bd671f1 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 11 Dec 2024 14:20:01 -0500 Subject: [PATCH 083/249] offload logging Signed-off-by: Vladimir Mandic --- TODO.md | 5 +++ modules/devices.py | 2 ++ modules/processing_diffusers.py | 1 + modules/sd_models.py | 55 ++++++++++++++++++++++----------- modules/shared.py | 2 +- 5 files changed, 46 insertions(+), 19 deletions(-) diff --git a/TODO.md b/TODO.md index 90372e41f..9692b7635 100644 --- a/TODO.md +++ b/TODO.md @@ -2,6 +2,11 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladmandic/projects) +## Pending + +- LoRA direct with caching +- Previewer issues + ## Future Candidates - SD35 IPAdapter: diff --git a/modules/devices.py b/modules/devices.py index 51b770481..3f1439fb7 100644 --- a/modules/devices.py +++ b/modules/devices.py @@ -231,6 +231,8 @@ def get_stats(): return gpu, ram t1 = time.time() timer.process.add('gc', t1 - t0) + if fast: + return gpu, ram new_gpu, new_used_gpu, new_ram, new_used_ram, oom = get_stats() before = { 'gpu': gpu, 'ram': ram } diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 9f12e44ab..3c59bbcf7 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -458,6 +458,7 @@ def process_diffusers(p: processing.StableDiffusionProcessing): extra_networks.deactivate(p) timer.process.add('lora', networks.timer.total) + networks.timer.clear(complete=True) results = process_decode(p, output) timer.process.record('decode') diff --git a/modules/sd_models.py b/modules/sd_models.py index 85a5dc5d7..c39c0263e 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -36,7 +36,6 @@ checkpoint_tiles = checkpoint_titles # legacy compatibility should_offload = ['sc', 'sd3', 'f1', 'hunyuandit', 'auraflow', 'omnigen'] offload_hook_instance = None -offload_component_map = {} class NoWatermark: @@ -367,11 +366,7 @@ def set_diffuser_offload(sd_model, op: str = 'model'): except Exception as e: shared.log.error(f'Setting {op}: offload={shared.opts.diffusers_offload_mode} {e}') if shared.opts.diffusers_offload_mode == "balanced": - try: - shared.log.debug(f'Setting {op}: offload={shared.opts.diffusers_offload_mode} watermarks low={shared.opts.diffusers_offload_min_gpu_memory} high={shared.opts.diffusers_offload_max_gpu_memory} limit={shared.opts.cuda_mem_fraction:.2f}') - sd_model = apply_balanced_offload(sd_model) - except Exception as e: - shared.log.error(f'Setting {op}: offload={shared.opts.diffusers_offload_mode} {e}') + sd_model = apply_balanced_offload(sd_model) process_timer.add('offload', time.time() - t0) @@ -386,11 +381,30 @@ def __init__(self): self.cpu_watermark = shared.opts.diffusers_offload_max_cpu_memory self.gpu = int(shared.gpu_memory * shared.opts.diffusers_offload_max_gpu_memory * 1024*1024*1024) self.cpu = int(shared.cpu_memory * shared.opts.diffusers_offload_max_cpu_memory * 1024*1024*1024) - gpu_dict = { "min": self.min_watermark, "max": self.max_watermark, "bytes": self.gpu } - cpu_dict = { "max": self.cpu_watermark, "bytes": self.cpu } - shared.log.info(f'Init offload: type=balanced gpu={gpu_dict} cpu={cpu_dict}') + self.offload_map = {} + gpu = f'{shared.gpu_memory * shared.opts.diffusers_offload_min_gpu_memory:.3f}-{shared.gpu_memory * shared.opts.diffusers_offload_max_gpu_memory}:{shared.gpu_memory}' + shared.log.info(f'Offload: type=balanced op=init watermark={self.min_watermark}-{self.max_watermark} gpu={gpu} cpu={shared.cpu_memory:.3f} limit={shared.opts.cuda_mem_fraction:.2f}') + self.validate() super().__init__() + def validate(self): + if shared.opts.diffusers_offload_mode != 'balanced': + return + if shared.opts.diffusers_offload_min_gpu_memory < 0 or shared.opts.diffusers_offload_min_gpu_memory > 1: + shared.opts.diffusers_offload_min_gpu_memory = 0.25 + shared.log.warning(f'Offload: type=balanced op=validate: watermark low={shared.opts.diffusers_offload_min_gpu_memory} invalid value') + if shared.opts.diffusers_offload_max_gpu_memory < 0.1 or shared.opts.diffusers_offload_max_gpu_memory > 1: + shared.opts.diffusers_offload_max_gpu_memory = 0.75 + shared.log.warning(f'Offload: type=balanced op=validate: watermark high={shared.opts.diffusers_offload_max_gpu_memory} invalid value') + if shared.opts.diffusers_offload_min_gpu_memory > shared.opts.diffusers_offload_max_gpu_memory: + shared.opts.diffusers_offload_min_gpu_memory = shared.opts.diffusers_offload_max_gpu_memory + shared.log.warning(f'Offload: type=balanced op=validate: watermark low={shared.opts.diffusers_offload_min_gpu_memory} reset') + if shared.opts.diffusers_offload_max_gpu_memory * shared.gpu_memory < 4: + shared.log.warning(f'Offload: type=balanced op=validate: watermark high={shared.opts.diffusers_offload_max_gpu_memory} low memory') + + def model_size(self): + return sum(self.offload_map.values()) + def init_hook(self, module): return module @@ -421,12 +435,14 @@ def apply_balanced_offload(sd_model, exclude=[]): global offload_hook_instance # pylint: disable=global-statement if shared.opts.diffusers_offload_mode != "balanced": return sd_model - if offload_hook_instance is None or offload_hook_instance.min_watermark != shared.opts.diffusers_offload_min_gpu_memory or offload_hook_instance.max_watermark != shared.opts.diffusers_offload_max_gpu_memory: - offload_hook_instance = OffloadHook() t0 = time.time() excluded = ['OmniGenPipeline'] if sd_model.__class__.__name__ in excluded: return sd_model + cached = True + if offload_hook_instance is None or offload_hook_instance.min_watermark != shared.opts.diffusers_offload_min_gpu_memory or offload_hook_instance.max_watermark != shared.opts.diffusers_offload_max_gpu_memory: + cached = False + offload_hook_instance = OffloadHook() checkpoint_name = sd_model.sd_checkpoint_info.name if getattr(sd_model, "sd_checkpoint_info", None) is not None else None if checkpoint_name is None: checkpoint_name = sd_model.__class__.__name__ @@ -439,7 +455,7 @@ def get_pipe_modules(pipe): modules_names = [m for m in modules_names if m not in exclude and not m.startswith('_')] modules = {} for module_name in modules_names: - module_size = offload_component_map.get(module_name, None) + module_size = offload_hook_instance.offload_map.get(module_name, None) if module_size is None: module = getattr(pipe, module_name, None) if not isinstance(module, torch.nn.Module): @@ -447,9 +463,9 @@ def get_pipe_modules(pipe): try: module_size = sum(p.numel()*p.element_size() for p in module.parameters(recurse=True)) / 1024 / 1024 / 1024 except Exception as e: - shared.log.error(f'Balanced offload: module={module_name} {e}') + shared.log.error(f'Offload: type=balanced op=calc module={module_name} {e}') module_size = 0 - offload_component_map[module_name] = module_size + offload_hook_instance.offload_map[module_name] = module_size modules[module_name] = module_size modules = sorted(modules.items(), key=lambda x: x[1], reverse=True) return modules @@ -476,12 +492,12 @@ def apply_balanced_offload_to_module(pipe): if do_offload: module = module.to(devices.cpu, non_blocking=True) used_gpu -= module_size - debug_move(f'Balanced offload: op={"move" if do_offload else "skip"} gpu={prev_gpu:.3f}:{used_gpu:.3f} perc={perc_gpu:.2f} ram={used_ram:.3f} current={module.device} dtype={module.dtype} component={module.__class__.__name__} size={module_size:.3f}') + debug_move(f'Offload: type=balanced op={"move" if do_offload else "skip"} gpu={prev_gpu:.3f}:{used_gpu:.3f} perc={perc_gpu:.2f} ram={used_ram:.3f} current={module.device} dtype={module.dtype} component={module.__class__.__name__} size={module_size:.3f}') except Exception as e: if 'bitsandbytes' not in str(e): - shared.log.error(f'Balanced offload: module={module_name} {e}') + shared.log.error(f'Offload: type=balanced op=apply module={module_name} {e}') if os.environ.get('SD_MOVE_DEBUG', None): - errors.display(e, f'Balanced offload: module={module_name}') + errors.display(e, f'Offload: type=balanced op=apply module={module_name}') module.offload_dir = os.path.join(shared.opts.accelerate_offload_path, checkpoint_name, module_name) module = accelerate.hooks.add_hook_to_module(module, offload_hook_instance, append=True) module._hf_hook.execution_device = torch.device(devices.device) # pylint: disable=protected-access @@ -505,6 +521,8 @@ def apply_balanced_offload_to_module(pipe): process_timer.add('offload', t) fn = f'{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access debug_move(f'Apply offload: time={t:.2f} type=balanced fn={fn}') + if not cached: + shared.log.info(f'Offload: type=balanced op=apply class={sd_model.__class__.__name__} modules={len(offload_hook_instance.offload_map)} size={offload_hook_instance.model_size():.3f}') return sd_model @@ -1000,7 +1018,8 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No shared.log.error(f"Load {op}: {e}") errors.display(e, "Model") - devices.torch_gc(force=True) + if shared.opts.diffusers_offload_mode != 'balanced': + devices.torch_gc(force=True) if sd_model is not None: script_callbacks.model_loaded_callback(sd_model) diff --git a/modules/shared.py b/modules/shared.py index ffba92442..7ed14f79a 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -484,7 +484,7 @@ def get_default_modes(): "diffusers_extract_ema": OptionInfo(False, "Use model EMA weights when possible", gr.Checkbox, {"visible": False }), "diffusers_offload_mode": OptionInfo(startup_offload_mode, "Model offload mode", gr.Radio, {"choices": ['none', 'balanced', 'model', 'sequential']}), "diffusers_offload_min_gpu_memory": OptionInfo(0.25, "Balanced offload GPU low watermark", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01 }), - "diffusers_offload_max_gpu_memory": OptionInfo(0.70, "Balanced offload GPU high watermark", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01 }), + "diffusers_offload_max_gpu_memory": OptionInfo(0.70, "Balanced offload GPU high watermark", gr.Slider, {"minimum": 0.1, "maximum": 1, "step": 0.01 }), "diffusers_offload_max_cpu_memory": OptionInfo(0.90, "Balanced offload CPU high watermark", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01, "visible": False }), "advanced_sep": OptionInfo("

Advanced Options

", "", gr.HTML), From eb9859c57634bc8e6ff284de805dadfa786af3f8 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 11 Dec 2024 15:22:51 -0500 Subject: [PATCH 084/249] update bnb and increase ui timeouts Signed-off-by: Vladimir Mandic --- TODO.md | 1 + installer.py | 6 +++--- javascript/logger.js | 10 ++++++---- modules/model_quant.py | 4 ++-- modules/sd_samplers_common.py | 3 ++- 5 files changed, 14 insertions(+), 10 deletions(-) diff --git a/TODO.md b/TODO.md index 9692b7635..76c672260 100644 --- a/TODO.md +++ b/TODO.md @@ -6,6 +6,7 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma - LoRA direct with caching - Previewer issues +- Redesign postprocessing ## Future Candidates diff --git a/installer.py b/installer.py index 93cd10413..b020418e1 100644 --- a/installer.py +++ b/installer.py @@ -682,7 +682,7 @@ def install_torch_addons(): if opts.get('nncf_compress_weights', False) and not args.use_openvino: install('nncf==2.7.0', 'nncf') if opts.get('optimum_quanto_weights', False): - install('optimum-quanto', 'optimum-quanto') + install('optimum-quanto==0.2.6', 'optimum-quanto') if triton_command is not None: install(triton_command, 'triton', quiet=True) @@ -999,8 +999,8 @@ def install_optional(): install('basicsr') install('gfpgan') install('clean-fid') - install('optimum-quanto', ignore=True) - install('bitsandbytes', ignore=True) + install('optimum-quanto=0.2.6', ignore=True) + install('bitsandbytes==0.45.0', ignore=True) install('pynvml', ignore=True) install('ultralytics==8.3.40', ignore=True) install('Cython', ignore=True) diff --git a/javascript/logger.js b/javascript/logger.js index 1677fa537..8fa812b86 100644 --- a/javascript/logger.js +++ b/javascript/logger.js @@ -1,3 +1,5 @@ +const timeout = 10000; + const log = async (...msg) => { const dt = new Date(); const ts = `${dt.getHours().toString().padStart(2, '0')}:${dt.getMinutes().toString().padStart(2, '0')}:${dt.getSeconds().toString().padStart(2, '0')}.${dt.getMilliseconds().toString().padStart(3, '0')}`; @@ -21,7 +23,7 @@ const error = async (...msg) => { // if (!txt.includes('asctime') && !txt.includes('xhr.')) xhrPost('/sdapi/v1/log', { error: txt }); // eslint-disable-line no-use-before-define }; -const xhrInternal = (xhrObj, data, handler = undefined, errorHandler = undefined, ignore = false, serverTimeout = 5000) => { +const xhrInternal = (xhrObj, data, handler = undefined, errorHandler = undefined, ignore = false, serverTimeout = timeout) => { const err = (msg) => { if (!ignore) { error(`${msg}: state=${xhrObj.readyState} status=${xhrObj.status} response=${xhrObj.responseText}`); @@ -30,7 +32,7 @@ const xhrInternal = (xhrObj, data, handler = undefined, errorHandler = undefined }; xhrObj.setRequestHeader('Content-Type', 'application/json'); - xhrObj.timeout = serverTimeout; + xhrObj.timeout = timeout; xhrObj.ontimeout = () => err('xhr.ontimeout'); xhrObj.onerror = () => err('xhr.onerror'); xhrObj.onabort = () => err('xhr.onabort'); @@ -52,14 +54,14 @@ const xhrInternal = (xhrObj, data, handler = undefined, errorHandler = undefined xhrObj.send(req); }; -const xhrGet = (url, data, handler = undefined, errorHandler = undefined, ignore = false, serverTimeout = 5000) => { +const xhrGet = (url, data, handler = undefined, errorHandler = undefined, ignore = false, serverTimeout = timeout) => { const xhr = new XMLHttpRequest(); const args = Object.keys(data).map((k) => `${encodeURIComponent(k)}=${encodeURIComponent(data[k])}`).join('&'); xhr.open('GET', `${url}?${args}`, true); xhrInternal(xhr, data, handler, errorHandler, ignore, serverTimeout); }; -function xhrPost(url, data, handler = undefined, errorHandler = undefined, ignore = false, serverTimeout = 5000) { +function xhrPost(url, data, handler = undefined, errorHandler = undefined, ignore = false, serverTimeout = timeout) { const xhr = new XMLHttpRequest(); xhr.open('POST', url, true); xhrInternal(xhr, data, handler, errorHandler, ignore, serverTimeout); diff --git a/modules/model_quant.py b/modules/model_quant.py index 03043b33a..5c0b40080 100644 --- a/modules/model_quant.py +++ b/modules/model_quant.py @@ -73,7 +73,7 @@ def load_bnb(msg='', silent=False): global bnb # pylint: disable=global-statement if bnb is not None: return bnb - install('bitsandbytes', quiet=True) + install('bitsandbytes==0.45.0', quiet=True) try: import bitsandbytes bnb = bitsandbytes @@ -96,7 +96,7 @@ def load_quanto(msg='', silent=False): global quanto # pylint: disable=global-statement if quanto is not None: return quanto - install('optimum-quanto', quiet=True) + install('optimum-quanto==0.2.6', quiet=True) try: from optimum import quanto as optimum_quanto # pylint: disable=no-name-in-module quanto = optimum_quanto diff --git a/modules/sd_samplers_common.py b/modules/sd_samplers_common.py index a96795a25..cd51043c7 100644 --- a/modules/sd_samplers_common.py +++ b/modules/sd_samplers_common.py @@ -51,6 +51,7 @@ def single_sample_to_image(sample, approximation=None): return Image.new(mode="RGB", size=(512, 512)) if len(sample.shape) == 4 and sample.shape[0]: # likely animatediff latent sample = sample.permute(1, 0, 2, 3)[0] + """ # TODO remove if shared.native: # [-x,x] to [-5,5] sample_max = torch.max(sample) @@ -59,7 +60,7 @@ def single_sample_to_image(sample, approximation=None): sample_min = torch.min(sample) if sample_min < -5: sample = sample * (5 / abs(sample_min)) - + """ if approximation == 2: # TAESD x_sample = sd_vae_taesd.decode(sample) x_sample = (1.0 + x_sample) / 2.0 # preview requires smaller range From e8a9fc2410cb08b91921ece2f623cf383bbefca7 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 11 Dec 2024 18:10:51 -0500 Subject: [PATCH 085/249] lora: absolute path, hf download, flux controlnet loras Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 9 +++- TODO.md | 1 + installer.py | 2 +- modules/control/run.py | 3 +- modules/control/units/controlnet.py | 73 ++++++++++++++++++++++------- modules/lora/networks.py | 16 +++++++ modules/processing_diffusers.py | 4 +- requirements.txt | 2 +- 8 files changed, 87 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f215b84ec..2ecc7ef26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Change Log for SD.Next -## Update for 2024-12-10 +## Update for 2024-12-11 ### New models and integrations @@ -22,6 +22,9 @@ *recommended*: guidance scale 30 - [Depth](https://huggingface.co/black-forest-labs/FLUX.1-Depth-dev): ~23.8GB, replaces currently loaded model *recommended*: guidance scale 10 +- [Flux ControlNet LoRA](https://huggingface.co/black-forest-labs/FLUX.1-Canny-dev-lora) + alternative to standard ControlNets, FLUX.1 also allows LoRA to help guide the generation process + both **Depth** and **Canny** LoRAs are available in standard control menus - [StabilityAI SD35 ControlNets]([sd3_medium](https://huggingface.co/stabilityai/stable-diffusion-3.5-controlnets)) - In addition to previously released `InstantX` and `Alimama`, we now have *official* ones from StabilityAI - [Style Aligned Image Generation](https://style-aligned-gen.github.io/) @@ -39,6 +42,10 @@ thanks @AI-Casanova - LoRA weights can be applied/unapplied as on each generate or they can store weights backups for later use this setting has large performance and resource implications, see [Offload](https://github.com/vladmandic/automatic/wiki/Offload) wiki for details + - LoRA name in prompt can now also be an absolute path to a LoRA file, even if LoRA is not indexed + example: `` + - LoRA name in prompt can now also be path to a LoRA file op `huggingface` + example: `` - **Model loader** improvements: - detect model components on model load fail - allow passing absolute path to model loader diff --git a/TODO.md b/TODO.md index 76c672260..63088d39f 100644 --- a/TODO.md +++ b/TODO.md @@ -17,6 +17,7 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma - SANA: - LTX-Video: - TorchAO: +- ControlNetUnion/ControlNetPromax: ## Other diff --git a/installer.py b/installer.py index b020418e1..36254b66a 100644 --- a/installer.py +++ b/installer.py @@ -459,7 +459,7 @@ def check_python(supported_minors=[9, 10, 11, 12], reason=None): def check_diffusers(): if args.skip_all or args.skip_requirements: return - sha = '3335e2262d47e7d7e311a44dea7f454b5f01b643' # diffusers commit hash + sha = '914a585be8187ec0ad92fab4f072c992f8c297cd' # diffusers commit hash pkg = pkg_resources.working_set.by_key.get('diffusers', None) minor = int(pkg.version.split('.')[1] if pkg is not None else 0) cur = opts.get('diffusers_version', '') if minor > 0 else '' diff --git a/modules/control/run.py b/modules/control/run.py index 2fe13dd73..6ae7fb20c 100644 --- a/modules/control/run.py +++ b/modules/control/run.py @@ -332,7 +332,7 @@ def set_pipe(): p.task_args['control_guidance_start'] = control_guidance_start p.task_args['control_guidance_end'] = control_guidance_end p.task_args['guess_mode'] = p.guess_mode - instance = controlnet.ControlNetPipeline(selected_models, shared.sd_model) + instance = controlnet.ControlNetPipeline(selected_models, shared.sd_model, p=p) pipe = instance.pipeline elif unit_type == 'xs' and has_models: p.extra_generation_params["Control mode"] = 'ControlNet-XS' @@ -370,7 +370,6 @@ def set_pipe(): debug(f'Control: run type={unit_type} models={has_models} pipe={pipe.__class__.__name__ if pipe is not None else None}') return pipe - pipe = set_pipe() debug(f'Control pipeline: class={pipe.__class__.__name__} args={vars(p)}') t1, t2, t3 = time.time(), 0, 0 diff --git a/modules/control/units/controlnet.py b/modules/control/units/controlnet.py index 3f68a4896..3fa2d90eb 100644 --- a/modules/control/units/controlnet.py +++ b/modules/control/units/controlnet.py @@ -5,6 +5,7 @@ from modules.control.units import detect from modules.shared import log, opts, listdir from modules import errors, sd_models, devices, model_quant +from modules.processing import StableDiffusionProcessingControl what = 'ControlNet' @@ -75,6 +76,8 @@ "InstantX Union": 'InstantX/FLUX.1-dev-Controlnet-Union', "InstantX Canny": 'InstantX/FLUX.1-dev-Controlnet-Canny', "JasperAI Depth": 'jasperai/Flux.1-dev-Controlnet-Depth', + "BlackForrestLabs Canny LoRA": '/huggingface.co/black-forest-labs/FLUX.1-Canny-dev-lora/flux1-canny-dev-lora.safetensors', + "BlackForrestLabs Depth LoRA": '/huggingface.co/black-forest-labs/FLUX.1-Depth-dev-lora/flux1-depth-dev-lora.safetensors', "JasperAI Surface Normals": 'jasperai/Flux.1-dev-Controlnet-Surface-Normals', "JasperAI Upscaler": 'jasperai/Flux.1-dev-Controlnet-Upscaler', "Shakker-Labs Union": 'Shakker-Labs/FLUX.1-dev-ControlNet-Union-Pro', @@ -162,12 +165,21 @@ def reset(self): self.model = None self.model_id = None - def get_class(self): + def get_class(self, model_id:str=''): import modules.shared if modules.shared.sd_model_type == 'sd': from diffusers import ControlNetModel as cls # pylint: disable=reimported config = 'lllyasviel/control_v11p_sd15_canny' elif modules.shared.sd_model_type == 'sdxl': + # TODO ControlNetUnion + """ + if 'union' in model_id.lower(): + from diffusers import ControlNetUnionModel as cls + config = 'xinsir/controlnet-union-sdxl-1.0' + else: + from diffusers import ControlNetModel as cls # pylint: disable=reimported # sdxl shares same model class + config = 'Eugeoter/noob-sdxl-controlnet-canny' + """ from diffusers import ControlNetModel as cls # pylint: disable=reimported # sdxl shares same model class config = 'Eugeoter/noob-sdxl-controlnet-canny' elif modules.shared.sd_model_type == 'f1': @@ -181,7 +193,7 @@ def get_class(self): return None, None return cls, config - def load_safetensors(self, model_path): + def load_safetensors(self, model_id, model_path): name = os.path.splitext(model_path)[0] config_path = None if not os.path.exists(model_path): @@ -206,7 +218,7 @@ def load_safetensors(self, model_path): config_path = f'{name}.json' if config_path is not None: self.load_config['original_config_file '] = config_path - cls, config = self.get_class() + cls, config = self.get_class(model_id) if cls is None: log.error(f'Control {what} model load failed: unknown base model') else: @@ -228,18 +240,21 @@ def load(self, model_id: str = None, force: bool = True) -> str: if model_path is None: log.error(f'Control {what} model load failed: id="{model_id}" error=unknown model id') return + if 'lora' in model_id.lower(): + self.model = model_path + return if model_id == self.model_id and not force: log.debug(f'Control {what} model: id="{model_id}" path="{model_path}" already loaded') return log.debug(f'Control {what} model loading: id="{model_id}" path="{model_path}"') + cls, _config = self.get_class(model_id) if model_path.endswith('.safetensors'): - self.load_safetensors(model_path) + self.load_safetensors(model_id, model_path) else: kwargs = {} if '/bin' in model_path: model_path = model_path.replace('/bin', '') self.load_config['use_safetensors'] = False - cls, _config = self.get_class() if cls is None: log.error(f'Control {what} model load failed: id="{model_id}" unknown base model') return @@ -271,7 +286,7 @@ def load(self, model_id: str = None, force: bool = True) -> str: self.model.to(self.device) t1 = time.time() self.model_id = model_id - log.debug(f'Control {what} model loaded: id="{model_id}" path="{model_path}" time={t1-t0:.2f}') + log.debug(f'Control {what} model loaded: id="{model_id}" path="{model_path}" cls={cls.__name__} time={t1-t0:.2f}') return f'{what} loaded model: {model_id}' except Exception as e: log.error(f'Control {what} model load failed: id="{model_id}" error={e}') @@ -284,16 +299,30 @@ def __init__(self, controlnet: Union[ControlNetModel, list[ControlNetModel]], pipeline: Union[StableDiffusionXLPipeline, StableDiffusionPipeline, FluxPipeline, StableDiffusion3Pipeline], dtype = None, + p: StableDiffusionProcessingControl = None, ): t0 = time.time() self.orig_pipeline = pipeline self.pipeline = None + + controlnets = controlnet if isinstance(controlnet, list) else [controlnet] + loras = [cn for cn in controlnets if isinstance(cn, str)] + controlnets = [cn for cn in controlnets if not isinstance(cn, str)] + if pipeline is None: log.error('Control model pipeline: model not loaded') return - elif detect.is_sdxl(pipeline): - from diffusers import StableDiffusionXLControlNetPipeline - self.pipeline = StableDiffusionXLControlNetPipeline( + elif detect.is_sdxl(pipeline) and len(controlnets) > 0: + from diffusers import StableDiffusionXLControlNetPipeline, StableDiffusionXLControlNetUnionPipeline + # TODO ControlNetUnion + """ + if controlnet.__class__.__name__ == 'ControlNetUnionModel': + cls = StableDiffusionXLControlNetUnionPipeline + else: + cls = StableDiffusionXLControlNetPipeline + """ + cls = StableDiffusionXLControlNetPipeline + self.pipeline = cls( vae=pipeline.vae, text_encoder=pipeline.text_encoder, text_encoder_2=pipeline.text_encoder_2, @@ -302,9 +331,9 @@ def __init__(self, unet=pipeline.unet, scheduler=pipeline.scheduler, feature_extractor=getattr(pipeline, 'feature_extractor', None), - controlnet=controlnet, # can be a list + controlnet=controlnets, # can be a list ) - elif detect.is_sd15(pipeline): + elif detect.is_sd15(pipeline) and len(controlnets) > 0: from diffusers import StableDiffusionControlNetPipeline self.pipeline = StableDiffusionControlNetPipeline( vae=pipeline.vae, @@ -315,10 +344,10 @@ def __init__(self, feature_extractor=getattr(pipeline, 'feature_extractor', None), requires_safety_checker=False, safety_checker=None, - controlnet=controlnet, # can be a list + controlnet=controlnets, # can be a list ) sd_models.move_model(self.pipeline, pipeline.device) - elif detect.is_f1(pipeline): + elif detect.is_f1(pipeline) and len(controlnets) > 0: from diffusers import FluxControlNetPipeline self.pipeline = FluxControlNetPipeline( vae=pipeline.vae.to(devices.device), @@ -328,9 +357,9 @@ def __init__(self, tokenizer_2=pipeline.tokenizer_2, transformer=pipeline.transformer, scheduler=pipeline.scheduler, - controlnet=controlnet, # can be a list + controlnet=controlnets, # can be a list ) - elif detect.is_sd3(pipeline): + elif detect.is_sd3(pipeline) and len(controlnets) > 0: from diffusers import StableDiffusion3ControlNetPipeline self.pipeline = StableDiffusion3ControlNetPipeline( vae=pipeline.vae, @@ -342,8 +371,18 @@ def __init__(self, tokenizer_3=pipeline.tokenizer_3, transformer=pipeline.transformer, scheduler=pipeline.scheduler, - controlnet=controlnet, # can be a list + controlnet=controlnets, # can be a list ) + elif len(loras) > 0: + self.pipeline = pipeline + for lora in loras: + log.debug(f'Control {what} pipeline: lora="{lora}"') + lora = lora.replace('/huggingface.co/', '') + self.pipeline.load_lora_weights(lora) + """ + if p is not None: + p.prompt += f'' + """ else: log.error(f'Control {what} pipeline: class={pipeline.__class__.__name__} unsupported model type') return @@ -353,6 +392,7 @@ def __init__(self, return if dtype is not None: self.pipeline = self.pipeline.to(dtype) + if opts.diffusers_offload_mode == 'none': sd_models.move_model(self.pipeline, devices.device) from modules.sd_models import set_diffuser_offload @@ -362,5 +402,6 @@ def __init__(self, log.debug(f'Control {what} pipeline: class={self.pipeline.__class__.__name__} time={t1-t0:.2f}') def restore(self): + self.pipeline.unload_lora_weights() self.pipeline = None return self.orig_pipeline diff --git a/modules/lora/networks.py b/modules/lora/networks.py index 5d285af95..a38945072 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -207,11 +207,27 @@ def add_network(filename): shared.log.info(f'Available LoRAs: path="{shared.cmd_opts.lora_dir}" items={len(available_networks)} folders={len(forbidden_network_aliases)} time={t1 - t0:.2f}') +def network_download(name): + from huggingface_hub import hf_hub_download + if os.path.exists(name): + return network.NetworkOnDisk(name, name) + parts = name.split('/') + if len(parts) >= 5 and parts[1] == 'huggingface.co': + repo_id = f'{parts[2]}/{parts[3]}' + filename = '/'.join(parts[4:]) + fn = hf_hub_download(repo_id=repo_id, filename=filename, cache_dir=shared.opts.hfcache_dir) + return network.NetworkOnDisk(name, fn) + return None + + def network_load(names, te_multipliers=None, unet_multipliers=None, dyn_dims=None): networks_on_disk: list[network.NetworkOnDisk] = [available_network_aliases.get(name, None) for name in names] if any(x is None for x in networks_on_disk): list_available_networks() networks_on_disk: list[network.NetworkOnDisk] = [available_network_aliases.get(name, None) for name in names] + for i in range(len(names)): + if names[i].startswith('/'): + networks_on_disk[i] = network_download(names[i]) failed_to_load_networks = [] recompile_model = maybe_recompile_model(names, te_multipliers) diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 3c59bbcf7..3b6f228ba 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -375,10 +375,10 @@ def process_decode(p: processing.StableDiffusionProcessing, output): elif hasattr(output, 'images'): results = output.images else: - shared.log.warning('Processing returned no results') + shared.log.warning('Processing: no results') results = [] else: - shared.log.warning('Processing returned no results') + shared.log.warning('Processing: no results') results = [] return results diff --git a/requirements.txt b/requirements.txt index 52cc7b2d4..da8870468 100644 --- a/requirements.txt +++ b/requirements.txt @@ -34,7 +34,7 @@ pi-heif # versioned safetensors==0.4.5 tensordict==0.1.2 -peft==0.13.1 +peft==0.14.0 httpx==0.24.1 compel==2.0.3 torchsde==0.2.6 From 80937c1338de87f2f64653b071d49025a8806a52 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 12 Dec 2024 13:58:58 -0500 Subject: [PATCH 086/249] add docs reference Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 4 +++- README.md | 9 +++++---- wiki | 2 +- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ecc7ef26..75979cc4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Change Log for SD.Next -## Update for 2024-12-11 +## Update for 2024-12-12 ### New models and integrations @@ -36,6 +36,8 @@ ### UI and workflow improvements +- **Docs**: + - New documentation site! - **LoRA** handler rewrite: - LoRA weights are no longer calculated on-the-fly during model execution, but are pre-calculated at the start this results in perceived overhead on generate startup, but results in overall faster execution as LoRA does not need to be processed on each step diff --git a/README.md b/README.md index 1bb5eacd0..722041c93 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@
-SD.Next +SD.Next **Image Diffusion implementation with advanced features** @@ -8,13 +8,14 @@ [![Discord](https://img.shields.io/discord/1101998836328697867?logo=Discord&svg=true)](https://discord.gg/VjvR2tabEX) [![Sponsors](https://img.shields.io/static/v1?label=Sponsor&message=%E2%9D%A4&logo=GitHub&color=%23fe8e86)](https://github.com/sponsors/vladmandic) -[Wiki](https://github.com/vladmandic/automatic/wiki) | [Discord](https://discord.gg/VjvR2tabEX) | [Changelog](CHANGELOG.md) +[Docs](https://vladmandic.github.io/sdnext-docs/) | [Wiki](https://github.com/vladmandic/automatic/wiki) | [Discord](https://discord.gg/VjvR2tabEX) | [Changelog](CHANGELOG.md)

## Table of contents +- [Documentation](https://vladmandic.github.io/sdnext-docs/) - [SD.Next Features](#sdnext-features) - [Model support](#model-support) - [Platform support](#platform-support) @@ -137,7 +138,7 @@ This should be fully cross-platform, but we'd really love to have additional con ### Credits -- Main credit goes to [Automatic1111 WebUI](https://github.com/AUTOMATIC1111/stable-diffusion-webui) for original codebase +- Main credit goes to [Automatic1111 WebUI](https://github.com/AUTOMATIC1111/stable-diffusion-webui) for the original codebase - Additional credits are listed in [Credits](https://github.com/AUTOMATIC1111/stable-diffusion-webui/#credits) - Licenses for modules are listed in [Licenses](html/licenses.html) @@ -154,7 +155,7 @@ This should be fully cross-platform, but we'd really love to have additional con ### Docs -If you're unsure how to use a feature, best place to start is [Wiki](https://github.com/vladmandic/automatic/wiki) and if its not there, +If you're unsure how to use a feature, best place to start is [Docs](https://vladmandic.github.io/sdnext-docs/) or [Wiki](https://github.com/vladmandic/automatic/wiki) and if its not there, check [ChangeLog](CHANGELOG.md) for when feature was first introduced as it will always have a short note on how to use it ### Sponsors diff --git a/wiki b/wiki index db828893c..8d63a0f04 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit db828893c803f1d5d0180cfe09689884bf27af2d +Subproject commit 8d63a0f04687f24c4ef413f231970087f167175c From e0895db442a441d0d29b01c7783897d8fef4df1a Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 13 Dec 2024 00:12:40 +0300 Subject: [PATCH 087/249] IPEX fix Flux --- modules/intel/ipex/diffusers.py | 42 +++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/modules/intel/ipex/diffusers.py b/modules/intel/ipex/diffusers.py index f742fe5c0..2af602558 100644 --- a/modules/intel/ipex/diffusers.py +++ b/modules/intel/ipex/diffusers.py @@ -20,20 +20,31 @@ def fourier_filter(x_in, threshold, scale): # fp64 error -def rope(pos: torch.Tensor, dim: int, theta: int) -> torch.Tensor: - assert dim % 2 == 0, "The dimension must be even." - - scale = torch.arange(0, dim, 2, dtype=torch.float32, device=pos.device) / dim # force fp32 instead of fp64 - omega = 1.0 / (theta**scale) - - batch_size, seq_length = pos.shape - out = torch.einsum("...n,d->...nd", pos, omega) - cos_out = torch.cos(out) - sin_out = torch.sin(out) - - stacked_out = torch.stack([cos_out, -sin_out, sin_out, cos_out], dim=-1) - out = stacked_out.view(batch_size, -1, dim // 2, 2, 2) - return out.float() +class FluxPosEmbed(torch.nn.Module): + def __init__(self, theta: int, axes_dim): + super().__init__() + self.theta = theta + self.axes_dim = axes_dim + + def forward(self, ids: torch.Tensor) -> torch.Tensor: + n_axes = ids.shape[-1] + cos_out = [] + sin_out = [] + pos = ids.float() + for i in range(n_axes): + cos, sin = diffusers.models.embeddings.get_1d_rotary_pos_embed( + self.axes_dim[i], + pos[:, i], + theta=self.theta, + repeat_interleave_real=True, + use_real=True, + freqs_dtype=torch.float32, + ) + cos_out.append(cos) + sin_out.append(sin) + freqs_cos = torch.cat(cos_out, dim=-1).to(ids.device) + freqs_sin = torch.cat(sin_out, dim=-1).to(ids.device) + return freqs_cos, freqs_sin @cache @@ -337,4 +348,5 @@ def ipex_diffusers(): if not device_supports_fp64 or os.environ.get('IPEX_FORCE_ATTENTION_SLICE', None) is not None: diffusers.models.attention_processor.SlicedAttnProcessor = SlicedAttnProcessor diffusers.models.attention_processor.AttnProcessor = AttnProcessor - diffusers.models.transformers.transformer_flux.rope = rope + if not device_supports_fp64: + diffusers.models.embeddings.FluxPosEmbed = FluxPosEmbed From 8adf5d0a3b9a527582ae24b304bb498ced775776 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 13 Dec 2024 22:05:13 +0300 Subject: [PATCH 088/249] Fix IPEX 2.5 --- modules/intel/ipex/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/intel/ipex/__init__.py b/modules/intel/ipex/__init__.py index e1c476e7e..147aa2798 100644 --- a/modules/intel/ipex/__init__.py +++ b/modules/intel/ipex/__init__.py @@ -77,7 +77,7 @@ def ipex_init(): # pylint: disable=too-many-statements torch.cuda.warnings = torch.xpu.warnings torch.cuda.classproperty = torch.xpu.classproperty torch.UntypedStorage.cuda = torch.UntypedStorage.xpu - if not ipex.__version__.startswith("2.3"): + if float(ipex.__version__[:3]) < 2.3: torch.cuda._initialization_lock = torch.xpu.lazy_init._initialization_lock torch.cuda._initialized = torch.xpu.lazy_init._initialized torch.cuda._is_in_bad_fork = torch.xpu.lazy_init._is_in_bad_fork @@ -111,7 +111,7 @@ def ipex_init(): # pylint: disable=too-many-statements torch.cuda.ComplexFloatStorage = torch.xpu.ComplexFloatStorage torch.cuda.ComplexDoubleStorage = torch.xpu.ComplexDoubleStorage - if not legacy or ipex.__version__.startswith("2.3"): + if not legacy or float(ipex.__version__[:3]) >= 2.3: torch.cuda._initialization_lock = torch.xpu._initialization_lock torch.cuda._initialized = torch.xpu._initialized torch.cuda._is_in_bad_fork = torch.xpu._is_in_bad_fork @@ -159,7 +159,7 @@ def ipex_init(): # pylint: disable=too-many-statements torch.xpu.amp.custom_fwd = torch.cuda.amp.custom_fwd torch.xpu.amp.custom_bwd = torch.cuda.amp.custom_bwd torch.cuda.amp = torch.xpu.amp - if not ipex.__version__.startswith("2.3"): + if float(ipex.__version__[:3]) < 2.3: torch.is_autocast_enabled = torch.xpu.is_autocast_xpu_enabled torch.get_autocast_gpu_dtype = torch.xpu.get_autocast_xpu_dtype @@ -178,7 +178,7 @@ def ipex_init(): # pylint: disable=too-many-statements torch.cuda.amp.GradScaler = ipex.cpu.autocast._grad_scaler.GradScaler # C - if legacy and not ipex.__version__.startswith("2.3"): + if legacy and float(ipex.__version__[:3]) < 2.3: torch._C._cuda_getCurrentRawStream = ipex._C._getCurrentRawStream ipex._C._DeviceProperties.multi_processor_count = ipex._C._DeviceProperties.gpu_subslice_count ipex._C._DeviceProperties.major = 12 From 08f919c24afc37aed23cb96c632b56a3dfe19de4 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 13 Dec 2024 22:16:56 +0300 Subject: [PATCH 089/249] Update to IPEX 2.5.10+xpu --- CHANGELOG.md | 1 + installer.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 75979cc4c..7b05eac4f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -83,6 +83,7 @@ ### Updates - Additional Wiki content: Styles, Wildcards, etc. +- **IPEX**: update to IPEX 2.5.10+xpu - **OpenVINO**: update to 2024.5.0 - **Sampler** improvements - Euler FlowMatch: add sigma methods (*karras/exponential/betas*) diff --git a/installer.py b/installer.py index 36254b66a..70c655ae3 100644 --- a/installer.py +++ b/installer.py @@ -635,13 +635,13 @@ def install_ipex(torch_command): if os.environ.get("ClDeviceGlobalMemSizeAvailablePercent", None) is None: os.environ.setdefault('ClDeviceGlobalMemSizeAvailablePercent', '100') if "linux" in sys.platform: - torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.3.1+cxx11.abi torchvision==0.18.1+cxx11.abi intel-extension-for-pytorch==2.3.110+xpu oneccl_bind_pt==2.3.100+xpu --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/') + torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.5.1+cxx11.abi torchvision==0.20.1+cxx11.abi intel-extension-for-pytorch==2.5.10+xpu oneccl_bind_pt==2.5.0+xpu --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/cn/') # torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision --index-url https://download.pytorch.org/whl/test/xpu') # test wheels are stable previews, significantly slower than IPEX # os.environ.setdefault('TENSORFLOW_PACKAGE', 'tensorflow==2.15.1 intel-extension-for-tensorflow[xpu]==2.15.0.1') else: torch_command = os.environ.get('TORCH_COMMAND', '--pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/xpu') # torchvision doesn't exist on test/stable branch for windows install(os.environ.get('OPENVINO_PACKAGE', 'openvino==2024.5.0'), 'openvino', ignore=True) - install('nncf==2.7.0', 'nncf', ignore=True) + install('nncf==2.7.0', ignore=True, no_deps=True) # requires older pandas install(os.environ.get('ONNXRUNTIME_PACKAGE', 'onnxruntime-openvino'), 'onnxruntime-openvino', ignore=True) return torch_command From c8272019c32f64fd21eb45fcd05ccf43d3b571dd Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 13 Dec 2024 23:01:53 +0300 Subject: [PATCH 090/249] Fix balanced offload with Cascade --- modules/intel/ipex/diffusers.py | 2 +- modules/sd_models.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/intel/ipex/diffusers.py b/modules/intel/ipex/diffusers.py index 2af602558..5bf5bbe39 100644 --- a/modules/intel/ipex/diffusers.py +++ b/modules/intel/ipex/diffusers.py @@ -1,7 +1,7 @@ import os from functools import wraps, cache import torch -import diffusers #0.29.1 # pylint: disable=import-error +import diffusers # pylint: disable=import-error from diffusers.models.attention_processor import Attention # pylint: disable=protected-access, missing-function-docstring, line-too-long diff --git a/modules/sd_models.py b/modules/sd_models.py index c39c0263e..c5875c61f 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -481,6 +481,8 @@ def apply_balanced_offload_to_module(pipe): keys = [k for k in keys if k not in exclude and not k.startswith('_')] for module_name, module_size in get_pipe_modules(pipe): # pylint: disable=protected-access module = getattr(pipe, module_name, None) + if module is None: + continue network_layer_name = getattr(module, "network_layer_name", None) device_map = getattr(module, "balanced_offload_device_map", None) max_memory = getattr(module, "balanced_offload_max_memory", None) From a5d0fa189da50c99521b544fe583fa45f3c7d6e9 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 14 Dec 2024 17:29:51 -0500 Subject: [PATCH 091/249] major controlnet work, xinsir promax and tiling support Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 17 +++++-- installer.py | 2 +- modules/control/run.py | 74 ++++++++++++++++++++-------- modules/control/tile.py | 75 +++++++++++++++++++++++++++++ modules/control/unit.py | 38 +++++++++++++-- modules/control/units/controlnet.py | 32 ++++++------ modules/images_resize.py | 10 ++-- modules/sd_models.py | 5 ++ modules/ui_control.py | 10 ++-- scripts/regional_prompting.py | 1 + 10 files changed, 208 insertions(+), 56 deletions(-) create mode 100644 modules/control/tile.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b05eac4f..c24484113 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Change Log for SD.Next -## Update for 2024-12-12 +## Update for 2024-12-13 ### New models and integrations @@ -33,11 +33,18 @@ style-aligned applies selected attention layers uniformly to all images to achive consistency can be used with or without input image in which case first prompt is used to establish baseline *note:* all prompts are processes as a single batch, so vram is limiting factor +- **ControlNet** + - improved support for `Union` controlnets with granular control mode type + - added support for latest [Xinsir ProMax](https://huggingface.co/xinsir/controlnet-union-sdxl-1.0) all-in-one controlnet + - added support for multiple **Tiling** controlnets, for example [Xinsir Tile](https://huggingface.co/xinsir/controlnet-tile-sdxl-1.0) + *note*: when selecting tiles in control settings, you can also specify non-square ratios + in which case it will use context-aware image resize to maintain overall composition ### UI and workflow improvements - **Docs**: - New documentation site! + - Additional Wiki content: Styles, Wildcards, etc. - **LoRA** handler rewrite: - LoRA weights are no longer calculated on-the-fly during model execution, but are pre-calculated at the start this results in perceived overhead on generate startup, but results in overall faster execution as LoRA does not need to be processed on each step @@ -82,7 +89,6 @@ ### Updates -- Additional Wiki content: Styles, Wildcards, etc. - **IPEX**: update to IPEX 2.5.10+xpu - **OpenVINO**: update to 2024.5.0 - **Sampler** improvements @@ -108,9 +114,10 @@ - simplify img2img/inpaint/sketch canvas handling - fix prompt caching - fix xyz grid skip final pass -- fix sd upscale script -- fix cogvideox-i2v -- lora auto-apply tags remove duplicates +- fix sd upscale script +- fix cogvideox-i2v +- lora auto-apply tags remove duplicates +- control load model on-demand if not already loaded ## Update for 2024-11-21 diff --git a/installer.py b/installer.py index 70c655ae3..18a8ad1f1 100644 --- a/installer.py +++ b/installer.py @@ -459,7 +459,7 @@ def check_python(supported_minors=[9, 10, 11, 12], reason=None): def check_diffusers(): if args.skip_all or args.skip_requirements: return - sha = '914a585be8187ec0ad92fab4f072c992f8c297cd' # diffusers commit hash + sha = '63243406ba5510c10d5cac931882918ceba926f9' # diffusers commit hash pkg = pkg_resources.working_set.by_key.get('diffusers', None) minor = int(pkg.version.split('.')[1] if pkg is not None else 0) cur = opts.get('diffusers_version', '') if minor > 0 else '' diff --git a/modules/control/run.py b/modules/control/run.py index 6ae7fb20c..ac1ff233d 100644 --- a/modules/control/run.py +++ b/modules/control/run.py @@ -7,6 +7,7 @@ from modules.control import util # helper functions from modules.control import unit # control units from modules.control import processors # image preprocessors +from modules.control import tile # tiling module from modules.control.units import controlnet # lllyasviel ControlNet from modules.control.units import xs # VisLearn ControlNet-XS from modules.control.units import lite # Kohya ControlLLLite @@ -83,6 +84,7 @@ def control_run(state: str = '', u.adapter.load(u.model_name, force=False) else: u.controlnet.load(u.model_name, force=False) + u.update_choices(u.model_name) if u.process is not None and u.process.override is None and u.override is not None: u.process.override = u.override @@ -149,6 +151,7 @@ def control_run(state: str = '', outpath_grids=shared.opts.outdir_grids or shared.opts.outdir_control_grids, ) p.state = state + p.is_tile = False # processing.process_init(p) resize_mode_before = resize_mode_before if resize_name_before != 'None' and inputs is not None and len(inputs) > 0 else 0 @@ -242,7 +245,7 @@ def control_run(state: str = '', active_model.append(u.adapter) active_strength.append(float(u.strength)) p.adapter_conditioning_factor = u.factor - shared.log.debug(f'Control T2I-Adapter unit: i={num_units} process={u.process.processor_id} model={u.adapter.model_id} strength={u.strength} factor={u.factor}') + shared.log.debug(f'Control T2I-Adapter unit: i={num_units} process="{u.process.processor_id}" model="{u.adapter.model_id}" strength={u.strength} factor={u.factor}') elif unit_type == 'controlnet' and u.controlnet.model is not None: active_process.append(u.process) active_model.append(u.controlnet) @@ -250,8 +253,12 @@ def control_run(state: str = '', active_start.append(float(u.start)) active_end.append(float(u.end)) p.guess_mode = u.guess - p.control_mode = u.mode - shared.log.debug(f'Control ControlNet unit: i={num_units} process={u.process.processor_id} model={u.controlnet.model_id} strength={u.strength} guess={u.guess} start={u.start} end={u.end} mode={u.mode}') + if isinstance(u.mode, str): + p.control_mode = u.choices.index(u.mode) if u.mode in u.choices else 0 + p.is_tile = p.is_tile or 'tile' in u.mode.lower() + p.control_tile = u.tile + p.extra_generation_params["Control mode"] = u.mode + shared.log.debug(f'Control ControlNet unit: i={num_units} process="{u.process.processor_id}" model="{u.controlnet.model_id}" strength={u.strength} guess={u.guess} start={u.start} end={u.end} mode={u.mode}') elif unit_type == 'xs' and u.controlnet.model is not None: active_process.append(u.process) active_model.append(u.controlnet) @@ -291,6 +298,7 @@ def control_run(state: str = '', selected_models = None elif len(active_model) == 1: selected_models = active_model[0].model if active_model[0].model is not None else None + p.is_tile = p.is_tile or 'tile' in active_model[0].model_id.lower() has_models = selected_models is not None control_conditioning = active_strength[0] if len(active_strength) > 0 else 1 # strength or list[strength] control_guidance_start = active_start[0] if len(active_start) > 0 else 0 @@ -305,29 +313,30 @@ def control_run(state: str = '', has_models = any(u.enabled for u in units if u.type == 'reference') else: pass + p.is_tile = p.is_tile and has_models def set_pipe(): global pipe, instance # pylint: disable=global-statement pipe = None if has_models: p.ops.append('control') - p.extra_generation_params["Control mode"] = unit_type # overriden later with pretty-print + p.extra_generation_params["Control type"] = unit_type # overriden later with pretty-print + p.extra_generation_params["Control model"] = ';'.join([(m.model_id or '') for m in active_model if m.model is not None]) p.extra_generation_params["Control conditioning"] = control_conditioning if isinstance(control_conditioning, list) else [control_conditioning] p.extra_generation_params['Control start'] = control_guidance_start if isinstance(control_guidance_start, list) else [control_guidance_start] p.extra_generation_params['Control end'] = control_guidance_end if isinstance(control_guidance_end, list) else [control_guidance_end] - p.extra_generation_params["Control model"] = ';'.join([(m.model_id or '') for m in active_model if m.model is not None]) p.extra_generation_params["Control conditioning"] = ';'.join([str(c) for c in p.extra_generation_params["Control conditioning"]]) p.extra_generation_params['Control start'] = ';'.join([str(c) for c in p.extra_generation_params['Control start']]) p.extra_generation_params['Control end'] = ';'.join([str(c) for c in p.extra_generation_params['Control end']]) if unit_type == 't2i adapter' and has_models: - p.extra_generation_params["Control mode"] = 'T2I-Adapter' + p.extra_generation_params["Control type"] = 'T2I-Adapter' p.task_args['adapter_conditioning_scale'] = control_conditioning instance = t2iadapter.AdapterPipeline(selected_models, shared.sd_model) pipe = instance.pipeline if inits is not None: shared.log.warning('Control: T2I-Adapter does not support separate init image') elif unit_type == 'controlnet' and has_models: - p.extra_generation_params["Control mode"] = 'ControlNet' + p.extra_generation_params["Control type"] = 'ControlNet' p.task_args['controlnet_conditioning_scale'] = control_conditioning p.task_args['control_guidance_start'] = control_guidance_start p.task_args['control_guidance_end'] = control_guidance_end @@ -335,7 +344,7 @@ def set_pipe(): instance = controlnet.ControlNetPipeline(selected_models, shared.sd_model, p=p) pipe = instance.pipeline elif unit_type == 'xs' and has_models: - p.extra_generation_params["Control mode"] = 'ControlNet-XS' + p.extra_generation_params["Control type"] = 'ControlNet-XS' p.controlnet_conditioning_scale = control_conditioning p.control_guidance_start = control_guidance_start p.control_guidance_end = control_guidance_end @@ -344,14 +353,14 @@ def set_pipe(): if inits is not None: shared.log.warning('Control: ControlNet-XS does not support separate init image') elif unit_type == 'lite' and has_models: - p.extra_generation_params["Control mode"] = 'ControlLLLite' + p.extra_generation_params["Control type"] = 'ControlLLLite' p.controlnet_conditioning_scale = control_conditioning instance = lite.ControlLLitePipeline(shared.sd_model) pipe = instance.pipeline if inits is not None: shared.log.warning('Control: ControlLLLite does not support separate init image') elif unit_type == 'reference' and has_models: - p.extra_generation_params["Control mode"] = 'Reference' + p.extra_generation_params["Control type"] = 'Reference' p.extra_generation_params["Control attention"] = p.attention p.task_args['reference_attn'] = 'Attention' in p.attention p.task_args['reference_adain'] = 'Adain' in p.attention @@ -393,6 +402,8 @@ def set_pipe(): else: original_pipeline = None + possible = sd_models.get_call(pipe).keys() + try: with devices.inference_context(): if isinstance(inputs, str): # only video, the rest is a list @@ -562,19 +573,29 @@ def set_pipe(): return [], '', '', 'Reference mode without image' elif unit_type == 'controlnet' and has_models: if input_type == 0: # Control only - if shared.sd_model_type in ['f1', 'sd3'] and 'control_image' not in p.task_args: - p.task_args['control_image'] = p.init_images # some controlnets mandate this + if 'control_image' in possible: + p.task_args['control_image'] = [p.init_images] if isinstance(p.init_images, Image.Image) else p.init_images + elif 'image' in possible: + p.task_args['image'] = [p.init_images] if isinstance(p.init_images, Image.Image) else p.init_images + if 'control_mode' in possible: + p.task_args['control_mode'] = p.control_mode + if 'strength' in possible: p.task_args['strength'] = p.denoising_strength + p.init_images = None elif input_type == 1: # Init image same as control - p.task_args['control_image'] = p.init_images # switch image and control_image - p.task_args['strength'] = p.denoising_strength + if 'control_image' in possible: + p.task_args['control_image'] = p.init_images # switch image and control_image + if 'strength' in possible: + p.task_args['strength'] = p.denoising_strength p.init_images = [p.override or input_image] * len(active_model) elif input_type == 2: # Separate init image if init_image is None: shared.log.warning('Control: separate init image not provided') init_image = input_image - p.task_args['control_image'] = p.init_images # switch image and control_image - p.task_args['strength'] = p.denoising_strength + if 'control_image' in possible: + p.task_args['control_image'] = p.init_images # switch image and control_image + if 'strength' in possible: + p.task_args['strength'] = p.denoising_strength p.init_images = [init_image] * len(active_model) if is_generator: @@ -607,11 +628,11 @@ def set_pipe(): p.task_args['strength'] = denoising_strength p.image_mask = mask shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.INPAINTING) # only controlnet supports inpaint - elif 'control_image' in p.task_args: + if hasattr(p, 'init_images') and p.init_images is not None: shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE) # only controlnet supports img2img else: shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE) - if hasattr(p, 'init_images') and p.init_images is not None: + if hasattr(p, 'init_images') and p.init_images is not None and 'image' in possible: p.task_args['image'] = p.init_images # need to set explicitly for txt2img del p.init_images if unit_type == 'lite': @@ -624,9 +645,14 @@ def set_pipe(): # final check if has_models: - if unit_type in ['controlnet', 't2i adapter', 'lite', 'xs'] and p.task_args.get('image', None) is None and getattr(p, 'init_images', None) is None: + if unit_type in ['controlnet', 't2i adapter', 'lite', 'xs'] \ + and p.task_args.get('image', None) is None \ + and p.task_args.get('control_image', None) is None \ + and getattr(p, 'init_images', None) is None \ + and getattr(p, 'image', None) is None: if is_generator: - yield terminate(f'Mode={p.extra_generation_params.get("Control mode", None)} input image is none') + shared.log.debug(f'Control args: {p.task_args}') + yield terminate(f'Mode={p.extra_generation_params.get("Control type", None)} input image is none') return [], '', '', 'Error: Input image is none' # resize mask @@ -656,11 +682,17 @@ def set_pipe(): script_runner.initialize_scripts(False) p.script_args = script.init_default_script_args(script_runner) - processed = p.scripts.run(p, *p.script_args) + # actual processing + if p.is_tile: + processed: processing.Processed = tile.run_tiling(p, input_image) + if processed is None: + processed = p.scripts.run(p, *p.script_args) if processed is None: processed: processing.Processed = processing.process_images(p) # run actual pipeline else: script_run = True + + # postprocessing processed = p.scripts.after(p, processed, *p.script_args) output = None if processed is not None: diff --git a/modules/control/tile.py b/modules/control/tile.py new file mode 100644 index 000000000..5dc104e47 --- /dev/null +++ b/modules/control/tile.py @@ -0,0 +1,75 @@ +from PIL import Image +from modules import shared, processing, images, sd_models + + +def get_tile(image: Image.Image, x: int, y: int, sx: int, sy: int) -> Image.Image: + return image.crop(( + (x + 0) * image.width // sx, + (y + 0) * image.height // sy, + (x + 1) * image.width // sx, + (y + 1) * image.height // sy + )) + + +def set_tile(image: Image.Image, x: int, y: int, tiled: Image.Image): + image.paste(tiled, (x * tiled.width, y * tiled.height)) + return image + + +def run_tiling(p: processing.StableDiffusionProcessing, input_image: Image.Image) -> processing.Processed: + # prepare images + sx, sy = p.control_tile.split('x') + sx = int(sx) + sy = int(sy) + if sx <= 0 or sy <= 0: + raise ValueError('Control: invalid tile size') + control_image = p.task_args.get('control_image', None) or p.task_args.get('image', None) + control_upscaled = None + if isinstance(control_image, list) and len(control_image) > 0: + control_upscaled = images.resize_image(resize_mode=1 if sx==sy else 5, + im=control_image[0], + width=8 * int(sx * control_image[0].width) // 8, + height=8 * int(sy * control_image[0].height) // 8, + context='add with forward' + ) + init_image = p.override or input_image + init_upscaled = None + if init_image is not None: + init_upscaled = images.resize_image(resize_mode=1 if sx==sy else 5, + im=init_image, + width=8 * int(sx * init_image.width) // 8, + height=8 * int(sy * init_image.height) // 8, + context='add with forward' + ) + + # stop processing from restoring pipeline on each iteration + orig_restore_pipeline = getattr(shared.sd_model, 'restore_pipeline', None) + shared.sd_model.restore_pipeline = None + + # run tiling + for x in range(sx): + for y in range(sy): + shared.log.info(f'Control Tile: tile={x+1}-{sx}/{y+1}-{sy} target={control_upscaled}') + shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE) + p.init_images = None + p.task_args['control_mode'] = p.control_mode + p.task_args['strength'] = p.denoising_strength + if init_upscaled is not None: + p.task_args['image'] = [get_tile(init_upscaled, x, y, sx, sy)] + if control_upscaled is not None: + p.task_args['control_image'] = [get_tile(control_upscaled, x, y, sx, sy)] + processed: processing.Processed = processing.process_images(p) # run actual pipeline + if processed is None or len(processed.images) == 0: + continue + control_upscaled = set_tile(control_upscaled, x, y, processed.images[0]) + + # post-process + p.width = control_upscaled.width + p.height = control_upscaled.height + processed.images = [control_upscaled] + processed.info = processed.infotext(p, 0) + processed.infotexts = [processed.info] + shared.sd_model.restore_pipeline = orig_restore_pipeline + if hasattr(shared.sd_model, 'restore_pipeline') and shared.sd_model.restore_pipeline is not None: + shared.sd_model.restore_pipeline() + return processed diff --git a/modules/control/unit.py b/modules/control/unit.py index 7dc5528a6..eeb729740 100644 --- a/modules/control/unit.py +++ b/modules/control/unit.py @@ -16,6 +16,22 @@ class Unit(): # mashup of gradio controls and mapping to actual implementation classes + def update_choices(self, model_id=None): + name = model_id or self.model_name + if name == 'InstantX Union': + self.choices = ['canny', 'tile', 'depth', 'blur', 'pose', 'gray', 'lq'] + elif name == 'Shakker-Labs Union': + self.choices = ['canny', 'tile', 'depth', 'blur', 'pose', 'gray', 'lq'] + elif name == 'Xinsir Union XL': + self.choices = ['openpose', 'depth', 'scribble', 'canny', 'normal'] + elif name == 'Xinsir ProMax XL': + self.choices = ['openpose', 'depth', 'scribble', 'canny', 'normal', 'segment', 'tile', 'repaint'] + else: + self.choices = ['default'] + + def __str__(self): + return f'Unit: type={self.type} enabled={self.enabled} strength={self.strength} start={self.start} end={self.end} mode={self.mode} tile={self.tile}' + def __init__(self, # values index: int = None, @@ -38,6 +54,7 @@ def __init__(self, control_start = None, control_end = None, control_mode = None, + control_tile = None, result_txt = None, extra_controls: list = [], ): @@ -70,6 +87,10 @@ def __init__(self, self.fidelity = 0.5 self.query_weight = 1.0 self.adain_weight = 1.0 + # control mode + self.choices = ['default'] + # control tile + self.tile = '1x1' def reset(): if self.process is not None: @@ -92,10 +113,16 @@ def control_change(start, end): self.end = max(start, end) def control_mode_change(mode): - self.mode = mode - 1 if mode > 0 else None + self.mode = self.choices.index(mode) if mode is not None and mode in self.choices else 0 + + def control_tile_change(tile): + self.tile = tile - def control_mode_show(model_id): - return gr.update(visible='union' in model_id.lower()) + def control_choices(model_id): + self.update_choices(model_id) + mode_visible = 'union' in model_id.lower() or 'promax' in model_id.lower() + tile_visible = 'union' in model_id.lower() or 'promax' in model_id.lower() or 'tile' in model_id.lower() + return [gr.update(visible=mode_visible, choices=self.choices), gr.update(visible=tile_visible)] def adapter_extra(c1): self.factor = c1 @@ -172,7 +199,7 @@ def set_image(image): else: self.controls.append(model_id) model_id.change(fn=self.controlnet.load, inputs=[model_id], outputs=[result_txt], show_progress=True) - model_id.change(fn=control_mode_show, inputs=[model_id], outputs=[control_mode], show_progress=False) + model_id.change(fn=control_choices, inputs=[model_id], outputs=[control_mode, control_tile], show_progress=False) if extra_controls is not None and len(extra_controls) > 0: extra_controls[0].change(fn=controlnet_extra, inputs=extra_controls) elif self.type == 'xs': @@ -231,3 +258,6 @@ def set_image(image): if control_mode is not None: self.controls.append(control_mode) control_mode.change(fn=control_mode_change, inputs=[control_mode]) + if control_tile is not None: + self.controls.append(control_tile) + control_tile.change(fn=control_tile_change, inputs=[control_tile]) diff --git a/modules/control/units/controlnet.py b/modules/control/units/controlnet.py index 3fa2d90eb..7361638c6 100644 --- a/modules/control/units/controlnet.py +++ b/modules/control/units/controlnet.py @@ -52,17 +52,20 @@ 'Depth Mid XL': 'diffusers/controlnet-depth-sdxl-1.0-mid', 'OpenPose XL': 'thibaud/controlnet-openpose-sdxl-1.0/bin', 'Xinsir Union XL': 'xinsir/controlnet-union-sdxl-1.0', + 'Xinsir ProMax XL': 'brad-twinkl/controlnet-union-sdxl-1.0-promax', 'Xinsir OpenPose XL': 'xinsir/controlnet-openpose-sdxl-1.0', 'Xinsir Canny XL': 'xinsir/controlnet-canny-sdxl-1.0', 'Xinsir Depth XL': 'xinsir/controlnet-depth-sdxl-1.0', 'Xinsir Scribble XL': 'xinsir/controlnet-scribble-sdxl-1.0', 'Xinsir Anime Painter XL': 'xinsir/anime-painter', + 'Xinsir Tile XL': 'xinsir/controlnet-tile-sdxl-1.0', 'NoobAI Canny XL': 'Eugeoter/noob-sdxl-controlnet-canny', 'NoobAI Lineart Anime XL': 'Eugeoter/noob-sdxl-controlnet-lineart_anime', 'NoobAI Depth XL': 'Eugeoter/noob-sdxl-controlnet-depth', 'NoobAI Normal XL': 'Eugeoter/noob-sdxl-controlnet-normal', 'NoobAI SoftEdge XL': 'Eugeoter/noob-sdxl-controlnet-softedge_hed', 'NoobAI OpenPose XL': 'einar77/noob-openpose', + 'TTPlanet Tile Realistic XL': 'Yakonrus/SDXL_Controlnet_Tile_Realistic_v2', # 'StabilityAI Canny R128': 'stabilityai/control-lora/control-LoRAs-rank128/control-lora-canny-rank128.safetensors', # 'StabilityAI Depth R128': 'stabilityai/control-lora/control-LoRAs-rank128/control-lora-depth-rank128.safetensors', # 'StabilityAI Recolor R128': 'stabilityai/control-lora/control-LoRAs-rank128/control-lora-recolor-rank128.safetensors', @@ -166,30 +169,30 @@ def reset(self): self.model_id = None def get_class(self, model_id:str=''): - import modules.shared - if modules.shared.sd_model_type == 'sd': + from modules import shared + if shared.sd_model_type == 'none': + _load = shared.sd_model # trigger a load + if shared.sd_model_type == 'sd': from diffusers import ControlNetModel as cls # pylint: disable=reimported config = 'lllyasviel/control_v11p_sd15_canny' - elif modules.shared.sd_model_type == 'sdxl': - # TODO ControlNetUnion - """ + elif shared.sd_model_type == 'sdxl': if 'union' in model_id.lower(): from diffusers import ControlNetUnionModel as cls config = 'xinsir/controlnet-union-sdxl-1.0' + elif 'promax' in model_id.lower(): + from diffusers import ControlNetUnionModel as cls + config = 'brad-twinkl/controlnet-union-sdxl-1.0-promax' else: from diffusers import ControlNetModel as cls # pylint: disable=reimported # sdxl shares same model class config = 'Eugeoter/noob-sdxl-controlnet-canny' - """ - from diffusers import ControlNetModel as cls # pylint: disable=reimported # sdxl shares same model class - config = 'Eugeoter/noob-sdxl-controlnet-canny' - elif modules.shared.sd_model_type == 'f1': + elif shared.sd_model_type == 'f1': from diffusers import FluxControlNetModel as cls config = 'InstantX/FLUX.1-dev-Controlnet-Union' - elif modules.shared.sd_model_type == 'sd3': + elif shared.sd_model_type == 'sd3': from diffusers import SD3ControlNetModel as cls config = 'InstantX/SD3-Controlnet-Canny' else: - log.error(f'Control {what}: type={modules.shared.sd_model_type} unsupported model') + log.error(f'Control {what}: type={shared.sd_model_type} unsupported model') return None, None return cls, config @@ -299,7 +302,7 @@ def __init__(self, controlnet: Union[ControlNetModel, list[ControlNetModel]], pipeline: Union[StableDiffusionXLPipeline, StableDiffusionPipeline, FluxPipeline, StableDiffusion3Pipeline], dtype = None, - p: StableDiffusionProcessingControl = None, + p: StableDiffusionProcessingControl = None, # pylint: disable=unused-argument ): t0 = time.time() self.orig_pipeline = pipeline @@ -314,14 +317,11 @@ def __init__(self, return elif detect.is_sdxl(pipeline) and len(controlnets) > 0: from diffusers import StableDiffusionXLControlNetPipeline, StableDiffusionXLControlNetUnionPipeline - # TODO ControlNetUnion - """ if controlnet.__class__.__name__ == 'ControlNetUnionModel': cls = StableDiffusionXLControlNetUnionPipeline + controlnets = controlnets[0] # using only first one else: cls = StableDiffusionXLControlNetPipeline - """ - cls = StableDiffusionXLControlNetPipeline self.pipeline = cls( vae=pipeline.vae, text_encoder=pipeline.text_encoder, diff --git a/modules/images_resize.py b/modules/images_resize.py index d86ff6f22..5cf3e57e4 100644 --- a/modules/images_resize.py +++ b/modules/images_resize.py @@ -5,7 +5,7 @@ from modules import shared -def resize_image(resize_mode, im, width, height, upscaler_name=None, output_type='image', context=None): +def resize_image(resize_mode: int, im: Image.Image, width: int, height: int, upscaler_name: str=None, output_type: str='image', context: str=None): upscaler_name = upscaler_name or shared.opts.upscaler_for_img2img def latent(im, w, h, upscaler): @@ -79,18 +79,18 @@ def fill(im, color=None): def context_aware(im, width, height, context): import seam_carving # https://github.com/li-plus/seam-carving - if 'forward' in context: + if 'forward' in context.lower(): energy_mode = "forward" - elif 'backward' in context: + elif 'backward' in context.lower(): energy_mode = "backward" else: return im - if 'Add' in context: + if 'add' in context.lower(): src_ratio = min(width / im.width, height / im.height) src_w = int(im.width * src_ratio) src_h = int(im.height * src_ratio) src_image = resize(im, src_w, src_h) - elif 'Remove' in context: + elif 'remove' in context.lower(): ratio = width / height src_ratio = im.width / im.height src_w = width if ratio > src_ratio else im.width * height // im.height diff --git a/modules/sd_models.py b/modules/sd_models.py index c5875c61f..5d42e314b 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -1057,6 +1057,11 @@ def get_signature(cls): return signature.parameters +def get_call(cls): + signature = inspect.signature(cls.__call__, follow_wrapped=True, eval_str=True) + return signature.parameters + + def switch_pipe(cls: diffusers.DiffusionPipeline, pipeline: diffusers.DiffusionPipeline = None, force = False, args = {}): """ args: diff --git a/modules/ui_control.py b/modules/ui_control.py index 59db12fc5..5a146a8fc 100644 --- a/modules/ui_control.py +++ b/modules/ui_control.py @@ -138,7 +138,7 @@ def create_ui(_blocks: gr.Blocks=None): show_input = gr.Checkbox(label="Show input", value=True, elem_id="control_show_input") show_preview = gr.Checkbox(label="Show preview", value=False, elem_id="control_show_preview") with gr.Row(): - input_type = gr.Radio(label="Input type", choices=['Control only', 'Init image same as control', 'Separate init image'], value='Control only', type='index', elem_id='control_input_type') + input_type = gr.Radio(label="Control input type", choices=['Control only', 'Init image same as control', 'Separate init image'], value='Control only', type='index', elem_id='control_input_type') with gr.Row(): denoising_strength = gr.Slider(minimum=0.01, maximum=1.0, step=0.01, label='Denoising strength', value=0.30, elem_id="control_input_denoising_strength") @@ -251,9 +251,10 @@ def create_ui(_blocks: gr.Blocks=None): model_id = gr.Dropdown(label="ControlNet", choices=controlnet.list_models(), value='None', elem_id=f'control_unit-{i}-model_name') ui_common.create_refresh_button(model_id, controlnet.list_models, lambda: {"choices": controlnet.list_models(refresh=True)}, f'refresh_controlnet_models_{i}') model_strength = gr.Slider(label="CN Strength", minimum=0.01, maximum=2.0, step=0.01, value=1.0, elem_id=f'control_unit-{i}-strength') - control_start = gr.Slider(label="Start", minimum=0.0, maximum=1.0, step=0.05, value=0, elem_id=f'control_unit-{i}-start') - control_end = gr.Slider(label="End", minimum=0.0, maximum=1.0, step=0.05, value=1.0, elem_id=f'control_unit-{i}-end') - control_mode = gr.Dropdown(label="CN Mode", choices=['', 'Canny', 'Tile', 'Depth', 'Blur', 'Pose', 'Gray', 'LQ'], value=0, type='index', visible=False, elem_id=f'control_unit-{i}-mode') + control_start = gr.Slider(label="CN Start", minimum=0.0, maximum=1.0, step=0.05, value=0, elem_id=f'control_unit-{i}-start') + control_end = gr.Slider(label="CN End", minimum=0.0, maximum=1.0, step=0.05, value=1.0, elem_id=f'control_unit-{i}-end') + control_mode = gr.Dropdown(label="CN Mode", choices=['default'], value='default', visible=False, elem_id=f'control_unit-{i}-mode') + control_tile = gr.Dropdown(label="CN Tiles", choices=['1x1', '1x2', '1x3', '1x4', '2x1', '2x1', '2x2', '2x3', '2x4', '3x1', '3x2', '3x3', '3x4', '4x1', '4x2', '4x3', '4x4'], value='1x1', visible=False, elem_id=f'control_unit-{i}-tile') reset_btn = ui_components.ToolButton(value=ui_symbols.reset) image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool']) image_reuse= ui_components.ToolButton(value=ui_symbols.reuse) @@ -278,6 +279,7 @@ def create_ui(_blocks: gr.Blocks=None): control_start = control_start, control_end = control_end, control_mode = control_mode, + control_tile = control_tile, extra_controls = extra_controls, ) ) diff --git a/scripts/regional_prompting.py b/scripts/regional_prompting.py index 08b84dd94..48309704e 100644 --- a/scripts/regional_prompting.py +++ b/scripts/regional_prompting.py @@ -82,6 +82,7 @@ def run(self, p: processing.StableDiffusionProcessing, mode, grid, power, thresh } # run pipeline shared.log.debug(f'Regional: args={p.task_args}') + p.task_args['prompt'] = p.prompt processed: processing.Processed = processing.process_images(p) # runs processing using main loop # restore pipeline and params From bcded9f5105bd6684fdf0d2d5732963c3db0ef15 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 15 Dec 2024 12:40:54 -0500 Subject: [PATCH 092/249] add freescale Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 16 +- modules/control/tile.py | 24 +- modules/control/units/controlnet.py | 12 +- modules/freescale/__init__.py | 4 + modules/freescale/free_lunch_utils.py | 305 ++++ modules/freescale/freescale_pipeline.py | 1189 ++++++++++++++++ .../freescale/freescale_pipeline_img2img.py | 1245 +++++++++++++++++ modules/freescale/scale_attention.py | 367 +++++ modules/processing_diffusers.py | 27 +- modules/sd_samplers_common.py | 4 +- modules/sd_vae_taesd.py | 3 + modules/shared.py | 1 + modules/shared_state.py | 4 +- modules/ui_control.py | 2 +- scripts/freescale.py | 130 ++ 15 files changed, 3303 insertions(+), 30 deletions(-) create mode 100644 modules/freescale/__init__.py create mode 100644 modules/freescale/free_lunch_utils.py create mode 100644 modules/freescale/freescale_pipeline.py create mode 100644 modules/freescale/freescale_pipeline_img2img.py create mode 100644 modules/freescale/scale_attention.py create mode 100644 scripts/freescale.py diff --git a/CHANGELOG.md b/CHANGELOG.md index c24484113..35d3fcbe9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Change Log for SD.Next -## Update for 2024-12-13 +## Update for 2024-12-15 ### New models and integrations @@ -32,13 +32,19 @@ enter multiple prompts in prompt field separated by new line style-aligned applies selected attention layers uniformly to all images to achive consistency can be used with or without input image in which case first prompt is used to establish baseline - *note:* all prompts are processes as a single batch, so vram is limiting factor + *note:* all prompts are processes as a single batch, so vram is limiting factor +- [FreeScale](https://github.com/ali-vilab/FreeScale) + enable in scripts, compatible with sd-xl for text and img2img + run iterative generation of images at different scales to achieve better results + can render 4k sdxl images + *note*: disable live preview to avoid memory issues when generating large images - **ControlNet** - - improved support for `Union` controlnets with granular control mode type + - improved support for **Union** controlnets with granular control mode type - added support for latest [Xinsir ProMax](https://huggingface.co/xinsir/controlnet-union-sdxl-1.0) all-in-one controlnet - added support for multiple **Tiling** controlnets, for example [Xinsir Tile](https://huggingface.co/xinsir/controlnet-tile-sdxl-1.0) *note*: when selecting tiles in control settings, you can also specify non-square ratios - in which case it will use context-aware image resize to maintain overall composition + in which case it will use context-aware image resize to maintain overall composition + *note*: available tiling options can be set in settings -> control ### UI and workflow improvements @@ -118,6 +124,8 @@ - fix cogvideox-i2v - lora auto-apply tags remove duplicates - control load model on-demand if not already loaded +- taesd limit render to 2024px +- taesd downscale preview to 1024px max ## Update for 2024-11-21 diff --git a/modules/control/tile.py b/modules/control/tile.py index 5dc104e47..de9df1131 100644 --- a/modules/control/tile.py +++ b/modules/control/tile.py @@ -1,3 +1,4 @@ +import time from PIL import Image from modules import shared, processing, images, sd_models @@ -17,30 +18,25 @@ def set_tile(image: Image.Image, x: int, y: int, tiled: Image.Image): def run_tiling(p: processing.StableDiffusionProcessing, input_image: Image.Image) -> processing.Processed: + t0 = time.time() # prepare images sx, sy = p.control_tile.split('x') sx = int(sx) sy = int(sy) if sx <= 0 or sy <= 0: - raise ValueError('Control: invalid tile size') + raise ValueError('Control Tile: invalid tile size') control_image = p.task_args.get('control_image', None) or p.task_args.get('image', None) control_upscaled = None if isinstance(control_image, list) and len(control_image) > 0: - control_upscaled = images.resize_image(resize_mode=1 if sx==sy else 5, - im=control_image[0], - width=8 * int(sx * control_image[0].width) // 8, - height=8 * int(sy * control_image[0].height) // 8, - context='add with forward' - ) + w, h = 8 * int(sx * control_image[0].width) // 8, 8 * int(sy * control_image[0].height) // 8 + control_upscaled = images.resize_image(resize_mode=1 if sx==sy else 5, im=control_image[0], width=w, height=h, context='add with forward') init_image = p.override or input_image init_upscaled = None if init_image is not None: - init_upscaled = images.resize_image(resize_mode=1 if sx==sy else 5, - im=init_image, - width=8 * int(sx * init_image.width) // 8, - height=8 * int(sy * init_image.height) // 8, - context='add with forward' - ) + w, h = 8 * int(sx * init_image.width) // 8, 8 * int(sy * init_image.height) // 8 + init_upscaled = images.resize_image(resize_mode=1 if sx==sy else 5, im=init_image, width=w, height=h, context='add with forward') + t1 = time.time() + shared.log.debug(f'Control Tile: scale={sx}x{sy} resize={"fixed" if sx==sy else "context"} control={control_upscaled} init={init_upscaled} time={t1-t0:.3f}') # stop processing from restoring pipeline on each iteration orig_restore_pipeline = getattr(shared.sd_model, 'restore_pipeline', None) @@ -72,4 +68,6 @@ def run_tiling(p: processing.StableDiffusionProcessing, input_image: Image.Image shared.sd_model.restore_pipeline = orig_restore_pipeline if hasattr(shared.sd_model, 'restore_pipeline') and shared.sd_model.restore_pipeline is not None: shared.sd_model.restore_pipeline() + t2 = time.time() + shared.log.debug(f'Control Tile: image={control_upscaled} time={t2-t0:.3f}') return processed diff --git a/modules/control/units/controlnet.py b/modules/control/units/controlnet.py index 7361638c6..c887aca8f 100644 --- a/modules/control/units/controlnet.py +++ b/modules/control/units/controlnet.py @@ -101,6 +101,14 @@ "Alimama Inpainting": 'alimama-creative/SD3-Controlnet-Inpainting', "Alimama SoftEdge": 'alimama-creative/SD3-Controlnet-Softedge', } +variants = { + 'NoobAI Canny XL': 'fp16', + 'NoobAI Lineart Anime XL': 'fp16', + 'NoobAI Depth XL': 'fp16', + 'NoobAI Normal XL': 'fp16', + 'NoobAI SoftEdge XL': 'fp16', + 'TTPlanet Tile Realistic XL': 'fp16', +} models = {} all_models = {} all_models.update(predefined_sd15) @@ -261,8 +269,8 @@ def load(self, model_id: str = None, force: bool = True) -> str: if cls is None: log.error(f'Control {what} model load failed: id="{model_id}" unknown base model') return - if 'Eugeoter' in model_path: - kwargs['variant'] = 'fp16' + if variants.get(model_id, None) is not None: + kwargs['variant'] = variants[model_id] self.model = cls.from_pretrained(model_path, **self.load_config, **kwargs) if self.model is None: return diff --git a/modules/freescale/__init__.py b/modules/freescale/__init__.py new file mode 100644 index 000000000..7b9c17f5d --- /dev/null +++ b/modules/freescale/__init__.py @@ -0,0 +1,4 @@ +# Credits: https://github.com/ali-vilab/FreeScale + +from .freescale_pipeline import StableDiffusionXLFreeScale +from .freescale_pipeline_img2img import StableDiffusionXLFreeScaleImg2Img diff --git a/modules/freescale/free_lunch_utils.py b/modules/freescale/free_lunch_utils.py new file mode 100644 index 000000000..be26b732a --- /dev/null +++ b/modules/freescale/free_lunch_utils.py @@ -0,0 +1,305 @@ +from typing import Any, Dict, Optional, Tuple +import torch +import torch.fft as fft +from diffusers.utils import is_torch_version + +""" Borrowed from https://github.com/ChenyangSi/FreeU/blob/main/demo/free_lunch_utils.py +""" + +def isinstance_str(x: object, cls_name: str): + """ + Checks whether x has any class *named* cls_name in its ancestry. + Doesn't require access to the class's implementation. + + Useful for patching! + """ + + for _cls in x.__class__.__mro__: + if _cls.__name__ == cls_name: + return True + + return False + + +def Fourier_filter(x, threshold, scale): + dtype = x.dtype + x = x.type(torch.float32) + # FFT + x_freq = fft.fftn(x, dim=(-2, -1)) + x_freq = fft.fftshift(x_freq, dim=(-2, -1)) + + B, C, H, W = x_freq.shape + mask = torch.ones((B, C, H, W)).cuda() + + crow, ccol = H // 2, W //2 + mask[..., crow - threshold:crow + threshold, ccol - threshold:ccol + threshold] = scale + x_freq = x_freq * mask + + # IFFT + x_freq = fft.ifftshift(x_freq, dim=(-2, -1)) + x_filtered = fft.ifftn(x_freq, dim=(-2, -1)).real + + x_filtered = x_filtered.type(dtype) + return x_filtered + + +def register_upblock2d(model): + def up_forward(self): + def forward(hidden_states, res_hidden_states_tuple, temb=None, upsample_size=None): + for resnet in self.resnets: + # pop res hidden states + res_hidden_states = res_hidden_states_tuple[-1] + res_hidden_states_tuple = res_hidden_states_tuple[:-1] + #print(f"in upblock2d, hidden states shape: {hidden_states.shape}") + hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1) + + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + if is_torch_version(">=", "1.11.0"): + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb, use_reentrant=False + ) + else: + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb + ) + else: + hidden_states = resnet(hidden_states, temb) + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states, upsample_size) + + return hidden_states + + return forward + + for i, upsample_block in enumerate(model.unet.up_blocks): + if isinstance_str(upsample_block, "UpBlock2D"): + upsample_block.forward = up_forward(upsample_block) + + +def register_free_upblock2d(model, b1=1.2, b2=1.4, s1=0.9, s2=0.2): + def up_forward(self): + def forward(hidden_states, res_hidden_states_tuple, temb=None, upsample_size=None): + for resnet in self.resnets: + # pop res hidden states + res_hidden_states = res_hidden_states_tuple[-1] + res_hidden_states_tuple = res_hidden_states_tuple[:-1] + #print(f"in free upblock2d, hidden states shape: {hidden_states.shape}") + + # --------------- FreeU code ----------------------- + # Only operate on the first two stages + if hidden_states.shape[1] == 1280: + hidden_states[:,:640] = hidden_states[:,:640] * self.b1 + res_hidden_states = Fourier_filter(res_hidden_states, threshold=1, scale=self.s1) + if hidden_states.shape[1] == 640: + hidden_states[:,:320] = hidden_states[:,:320] * self.b2 + res_hidden_states = Fourier_filter(res_hidden_states, threshold=1, scale=self.s2) + # --------------------------------------------------------- + + hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1) + + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + if is_torch_version(">=", "1.11.0"): + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb, use_reentrant=False + ) + else: + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb + ) + else: + hidden_states = resnet(hidden_states, temb) + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states, upsample_size) + + return hidden_states + + return forward + + for i, upsample_block in enumerate(model.unet.up_blocks): + if isinstance_str(upsample_block, "UpBlock2D"): + upsample_block.forward = up_forward(upsample_block) + setattr(upsample_block, 'b1', b1) + setattr(upsample_block, 'b2', b2) + setattr(upsample_block, 's1', s1) + setattr(upsample_block, 's2', s2) + + +def register_crossattn_upblock2d(model): + def up_forward(self): + def forward( + hidden_states: torch.FloatTensor, + res_hidden_states_tuple: Tuple[torch.FloatTensor, ...], + temb: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + upsample_size: Optional[int] = None, + attention_mask: Optional[torch.FloatTensor] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + ): + for resnet, attn in zip(self.resnets, self.attentions): + # pop res hidden states + #print(f"in crossatten upblock2d, hidden states shape: {hidden_states.shape}") + res_hidden_states = res_hidden_states_tuple[-1] + res_hidden_states_tuple = res_hidden_states_tuple[:-1] + hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1) + + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module, return_dict=None): + def custom_forward(*inputs): + if return_dict is not None: + return module(*inputs, return_dict=return_dict) + else: + return module(*inputs) + + return custom_forward + + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), + hidden_states, + temb, + **ckpt_kwargs, + ) + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(attn, return_dict=False), + hidden_states, + encoder_hidden_states, + None, # timestep + None, # class_labels + cross_attention_kwargs, + attention_mask, + encoder_attention_mask, + **ckpt_kwargs, + )[0] + else: + hidden_states = resnet(hidden_states, temb) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + attention_mask=attention_mask, + encoder_attention_mask=encoder_attention_mask, + return_dict=False, + )[0] + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states, upsample_size) + + return hidden_states + + return forward + + for i, upsample_block in enumerate(model.unet.up_blocks): + if isinstance_str(upsample_block, "CrossAttnUpBlock2D"): + upsample_block.forward = up_forward(upsample_block) + + +def register_free_crossattn_upblock2d(model, b1=1.2, b2=1.4, s1=0.9, s2=0.2): + def up_forward(self): + def forward( + hidden_states: torch.FloatTensor, + res_hidden_states_tuple: Tuple[torch.FloatTensor, ...], + temb: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + upsample_size: Optional[int] = None, + attention_mask: Optional[torch.FloatTensor] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + ): + for resnet, attn in zip(self.resnets, self.attentions): + # pop res hidden states + #print(f"in free crossatten upblock2d, hidden states shape: {hidden_states.shape}") + res_hidden_states = res_hidden_states_tuple[-1] + res_hidden_states_tuple = res_hidden_states_tuple[:-1] + + # --------------- FreeU code ----------------------- + # Only operate on the first two stages + if hidden_states.shape[1] == 1280: + hidden_states[:,:640] = hidden_states[:,:640] * self.b1 + res_hidden_states = Fourier_filter(res_hidden_states, threshold=1, scale=self.s1) + if hidden_states.shape[1] == 640: + hidden_states[:,:320] = hidden_states[:,:320] * self.b2 + res_hidden_states = Fourier_filter(res_hidden_states, threshold=1, scale=self.s2) + # --------------------------------------------------------- + + hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1) + + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module, return_dict=None): + def custom_forward(*inputs): + if return_dict is not None: + return module(*inputs, return_dict=return_dict) + else: + return module(*inputs) + + return custom_forward + + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), + hidden_states, + temb, + **ckpt_kwargs, + ) + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(attn, return_dict=False), + hidden_states, + encoder_hidden_states, + None, # timestep + None, # class_labels + cross_attention_kwargs, + attention_mask, + encoder_attention_mask, + **ckpt_kwargs, + )[0] + else: + hidden_states = resnet(hidden_states, temb) + # hidden_states = attn( + # hidden_states, + # encoder_hidden_states=encoder_hidden_states, + # cross_attention_kwargs=cross_attention_kwargs, + # encoder_attention_mask=encoder_attention_mask, + # return_dict=False, + # )[0] + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + )[0] + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states, upsample_size) + + return hidden_states + + return forward + + for i, upsample_block in enumerate(model.unet.up_blocks): + if isinstance_str(upsample_block, "CrossAttnUpBlock2D"): + upsample_block.forward = up_forward(upsample_block) + setattr(upsample_block, 'b1', b1) + setattr(upsample_block, 'b2', b2) + setattr(upsample_block, 's1', s1) + setattr(upsample_block, 's2', s2) diff --git a/modules/freescale/freescale_pipeline.py b/modules/freescale/freescale_pipeline.py new file mode 100644 index 000000000..9b7a68b68 --- /dev/null +++ b/modules/freescale/freescale_pipeline.py @@ -0,0 +1,1189 @@ +from inspect import isfunction +from functools import partial +from typing import Any, Callable, Dict, List, Optional, Tuple, Union +import inspect +import os +import random + +import numpy as np +import torch +import torch.nn.functional as F +from einops import rearrange +from transformers import CLIPTextModel, CLIPTextModelWithProjection, CLIPTokenizer + +from diffusers.image_processor import VaeImageProcessor +from diffusers.loaders import FromSingleFileMixin, LoraLoaderMixin, TextualInversionLoaderMixin +from diffusers.models import AutoencoderKL, UNet2DConditionModel +from diffusers.models.attention_processor import AttnProcessor2_0, LoRAAttnProcessor2_0, LoRAXFormersAttnProcessor, XFormersAttnProcessor +from diffusers.schedulers import KarrasDiffusionSchedulers +from diffusers.utils.torch_utils import randn_tensor +from diffusers.utils import is_accelerate_available, is_accelerate_version, logging, replace_example_docstring +from diffusers.pipelines.pipeline_utils import DiffusionPipeline +from diffusers.pipelines.stable_diffusion_xl.pipeline_output import StableDiffusionXLPipelineOutput +from diffusers.models.attention import BasicTransformerBlock + +from .scale_attention import ori_forward, scale_forward + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> from diffusers import StableDiffusionXLPipeline + + >>> pipe = StableDiffusionXLPipeline.from_pretrained( + ... "stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16 + ... ) + >>> pipe = pipe.to("cuda") + + >>> prompt = "a photo of an astronaut riding a horse on mars" + >>> image = pipe(prompt).images[0] + ``` +""" + +def default(val, d): + if exists(val): + return val + return d() if isfunction(d) else d + +def exists(val): + return val is not None + +def extract_into_tensor(a, t, x_shape): + b, *_ = t.shape + out = a.gather(-1, t) + return out.reshape(b, *((1,) * (len(x_shape) - 1))) + +def make_beta_schedule(schedule, n_timestep, linear_start=1e-4, linear_end=2e-2, cosine_s=8e-3): + if schedule == "linear": + betas = ( + torch.linspace(linear_start ** 0.5, linear_end ** 0.5, n_timestep, dtype=torch.float64) ** 2 + ) + elif schedule == "cosine": + timesteps = ( + torch.arange(n_timestep + 1, dtype=torch.float64) / n_timestep + cosine_s + ) + alphas = timesteps / (1 + cosine_s) * np.pi / 2 + alphas = torch.cos(alphas).pow(2) + alphas = alphas / alphas[0] + betas = 1 - alphas[1:] / alphas[:-1] + betas = np.clip(betas, a_min=0, a_max=0.999) + elif schedule == "sqrt_linear": + betas = torch.linspace(linear_start, linear_end, n_timestep, dtype=torch.float64) + elif schedule == "sqrt": + betas = torch.linspace(linear_start, linear_end, n_timestep, dtype=torch.float64) ** 0.5 + else: + raise ValueError(f"schedule '{schedule}' unknown.") + return betas.numpy() + +to_torch = partial(torch.tensor, dtype=torch.float16) +betas = make_beta_schedule("linear", 1000, linear_start=0.00085, linear_end=0.012) +alphas = 1. - betas +alphas_cumprod = np.cumprod(alphas, axis=0) +sqrt_alphas_cumprod = to_torch(np.sqrt(alphas_cumprod)) +sqrt_one_minus_alphas_cumprod = to_torch(np.sqrt(1. - alphas_cumprod)) + +def q_sample(x_start, t, init_noise_sigma = 1.0, noise=None, device=None): + noise = default(noise, lambda: torch.randn_like(x_start)).to(device) * init_noise_sigma + return (extract_into_tensor(sqrt_alphas_cumprod.to(device), t, x_start.shape) * x_start + + extract_into_tensor(sqrt_one_minus_alphas_cumprod.to(device), t, x_start.shape) * noise) + +def get_views(height, width, h_window_size=128, w_window_size=128, h_window_stride=64, w_window_stride=64, vae_scale_factor=8): + height //= vae_scale_factor + width //= vae_scale_factor + num_blocks_height = int((height - h_window_size) / h_window_stride - 1e-6) + 2 if height > h_window_size else 1 + num_blocks_width = int((width - w_window_size) / w_window_stride - 1e-6) + 2 if width > w_window_size else 1 + total_num_blocks = int(num_blocks_height * num_blocks_width) + views = [] + for i in range(total_num_blocks): + h_start = int((i // num_blocks_width) * h_window_stride) + h_end = h_start + h_window_size + w_start = int((i % num_blocks_width) * w_window_stride) + w_end = w_start + w_window_size + + if h_end > height: + h_start = int(h_start + height - h_end) + h_end = int(height) + if w_end > width: + w_start = int(w_start + width - w_end) + w_end = int(width) + if h_start < 0: + h_end = int(h_end - h_start) + h_start = 0 + if w_start < 0: + w_end = int(w_end - w_start) + w_start = 0 + + random_jitter = True + if random_jitter: + h_jitter_range = (h_window_size - h_window_stride) // 4 + w_jitter_range = (w_window_size - w_window_stride) // 4 + h_jitter = 0 + w_jitter = 0 + + if (w_start != 0) and (w_end != width): + w_jitter = random.randint(-w_jitter_range, w_jitter_range) + elif (w_start == 0) and (w_end != width): + w_jitter = random.randint(-w_jitter_range, 0) + elif (w_start != 0) and (w_end == width): + w_jitter = random.randint(0, w_jitter_range) + if (h_start != 0) and (h_end != height): + h_jitter = random.randint(-h_jitter_range, h_jitter_range) + elif (h_start == 0) and (h_end != height): + h_jitter = random.randint(-h_jitter_range, 0) + elif (h_start != 0) and (h_end == height): + h_jitter = random.randint(0, h_jitter_range) + h_start += (h_jitter + h_jitter_range) + h_end += (h_jitter + h_jitter_range) + w_start += (w_jitter + w_jitter_range) + w_end += (w_jitter + w_jitter_range) + + views.append((h_start, h_end, w_start, w_end)) + return views + +def gaussian_kernel(kernel_size=3, sigma=1.0, channels=3): + x_coord = torch.arange(kernel_size) + gaussian_1d = torch.exp(-(x_coord - (kernel_size - 1) / 2) ** 2 / (2 * sigma ** 2)) + gaussian_1d = gaussian_1d / gaussian_1d.sum() + gaussian_2d = gaussian_1d[:, None] * gaussian_1d[None, :] + kernel = gaussian_2d[None, None, :, :].repeat(channels, 1, 1, 1) + + return kernel + +def gaussian_filter(latents, kernel_size=3, sigma=1.0): + channels = latents.shape[1] + kernel = gaussian_kernel(kernel_size, sigma, channels).to(latents.device, latents.dtype) + if len(latents.shape) == 5: + b = latents.shape[0] + latents = rearrange(latents, 'b c t i j -> (b t) c i j') + blurred_latents = F.conv2d(latents, kernel, padding=kernel_size//2, groups=channels) + blurred_latents = rearrange(blurred_latents, '(b t) c i j -> b c t i j', b=b) + else: + blurred_latents = F.conv2d(latents, kernel, padding=kernel_size//2, groups=channels) + + return blurred_latents + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.rescale_noise_cfg +def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0): + """ + Rescale `noise_cfg` according to `guidance_rescale`. Based on findings of [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). See Section 3.4 + """ + std_text = noise_pred_text.std(dim=list(range(1, noise_pred_text.ndim)), keepdim=True) + std_cfg = noise_cfg.std(dim=list(range(1, noise_cfg.ndim)), keepdim=True) + # rescale the results from guidance (fixes overexposure) + noise_pred_rescaled = noise_cfg * (std_text / std_cfg) + # mix with the original results from guidance by factor guidance_rescale to avoid "plain looking" images + noise_cfg = guidance_rescale * noise_pred_rescaled + (1 - guidance_rescale) * noise_cfg + return noise_cfg + + +class StableDiffusionXLFreeScale(DiffusionPipeline, FromSingleFileMixin, LoraLoaderMixin): + r""" + Pipeline for text-to-image generation using Stable Diffusion XL. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + In addition the pipeline inherits the following loading methods: + - *LoRA*: [`StableDiffusionXLPipeline.load_lora_weights`] + - *Ckpt*: [`loaders.FromSingleFileMixin.from_single_file`] + + as well as the following saving methods: + - *LoRA*: [`loaders.StableDiffusionXLPipeline.save_lora_weights`] + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. + text_encoder ([`CLIPTextModel`]): + Frozen text-encoder. Stable Diffusion XL uses the text portion of + [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically + the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant. + text_encoder_2 ([` CLIPTextModelWithProjection`]): + Second frozen text-encoder. Stable Diffusion XL uses the text and pool portion of + [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModelWithProjection), + specifically the + [laion/CLIP-ViT-bigG-14-laion2B-39B-b160k](https://huggingface.co/laion/CLIP-ViT-bigG-14-laion2B-39B-b160k) + variant. + tokenizer (`CLIPTokenizer`): + Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). + tokenizer_2 (`CLIPTokenizer`): + Second Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). + unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + """ + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + text_encoder_2: CLIPTextModelWithProjection, + tokenizer: CLIPTokenizer, + tokenizer_2: CLIPTokenizer, + unet: UNet2DConditionModel, + scheduler: KarrasDiffusionSchedulers, + force_zeros_for_empty_prompt: bool = True, + ): + super().__init__() + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + text_encoder_2=text_encoder_2, + tokenizer=tokenizer, + tokenizer_2=tokenizer_2, + unet=unet, + scheduler=scheduler, + ) + self.register_to_config(force_zeros_for_empty_prompt=force_zeros_for_empty_prompt) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + self.default_sample_size = self.unet.config.sample_size + self.vae.enable_tiling() + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.enable_vae_slicing + def enable_vae_slicing(self): + r""" + Enable sliced VAE decoding. When this option is enabled, the VAE will split the input tensor in slices to + compute decoding in several steps. This is useful to save some memory and allow larger batch sizes. + """ + self.vae.enable_slicing() + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.disable_vae_slicing + def disable_vae_slicing(self): + r""" + Disable sliced VAE decoding. If `enable_vae_slicing` was previously enabled, this method will go back to + computing decoding in one step. + """ + self.vae.disable_slicing() + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.enable_vae_tiling + def enable_vae_tiling(self): + r""" + Enable tiled VAE decoding. When this option is enabled, the VAE will split the input tensor into tiles to + compute decoding and encoding in several steps. This is useful for saving a large amount of memory and to allow + processing larger images. + """ + self.vae.enable_tiling() + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.disable_vae_tiling + def disable_vae_tiling(self): + r""" + Disable tiled VAE decoding. If `enable_vae_tiling` was previously enabled, this method will go back to + computing decoding in one step. + """ + self.vae.disable_tiling() + + def enable_model_cpu_offload(self, gpu_id=0): + r""" + Offloads all models to CPU using accelerate, reducing memory usage with a low impact on performance. Compared + to `enable_sequential_cpu_offload`, this method moves one whole model at a time to the GPU when its `forward` + method is called, and the model remains in GPU until the next model runs. Memory savings are lower than with + `enable_sequential_cpu_offload`, but performance is much better due to the iterative execution of the `unet`. + """ + if is_accelerate_available() and is_accelerate_version(">=", "0.17.0.dev0"): + from accelerate import cpu_offload_with_hook + else: + raise ImportError("`enable_model_cpu_offload` requires `accelerate v0.17.0` or higher.") + + device = torch.device(f"cuda:{gpu_id}") + + if self.device.type != "cpu": + self.to("cpu", silence_dtype_warnings=True) + torch.cuda.empty_cache() # otherwise we don't see the memory savings (but they probably exist) + + model_sequence = ( + [self.text_encoder, self.text_encoder_2] if self.text_encoder is not None else [self.text_encoder_2] + ) + model_sequence.extend([self.unet, self.vae]) + + hook = None + for cpu_offloaded_model in model_sequence: + _, hook = cpu_offload_with_hook(cpu_offloaded_model, device, prev_module_hook=hook) + + # We'll offload the last model manually. + self.final_offload_hook = hook + + def encode_prompt( + self, + prompt: str, + prompt_2: Optional[str] = None, + device: Optional[torch.device] = None, + num_images_per_prompt: int = 1, + do_classifier_free_guidance: bool = True, + negative_prompt: Optional[str] = None, + negative_prompt_2: Optional[str] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is + used in both text-encoders + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + negative_prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and + `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. + If not provided, pooled text embeddings will be generated from `prompt` input argument. + negative_pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` + input argument. + lora_scale (`float`, *optional*): + A lora scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + """ + device = device or self._execution_device + + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, LoraLoaderMixin): + self._lora_scale = lora_scale + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + # Define tokenizers and text encoders + tokenizers = [self.tokenizer, self.tokenizer_2] if self.tokenizer is not None else [self.tokenizer_2] + text_encoders = ( + [self.text_encoder, self.text_encoder_2] if self.text_encoder is not None else [self.text_encoder_2] + ) + + if prompt_embeds is None: + prompt_2 = prompt_2 or prompt + # textual inversion: procecss multi-vector tokens if necessary + prompt_embeds_list = [] + prompts = [prompt, prompt_2] + for prompt, tokenizer, text_encoder in zip(prompts, tokenizers, text_encoders): + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, tokenizer) + + text_inputs = tokenizer( + prompt, + padding="max_length", + max_length=tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + + text_input_ids = text_inputs.input_ids + untruncated_ids = tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = tokenizer.batch_decode(untruncated_ids[:, tokenizer.model_max_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {tokenizer.model_max_length} tokens: {removed_text}" + ) + + prompt_embeds = text_encoder( + text_input_ids.to(device), + output_hidden_states=True, + ) + + # We are only ALWAYS interested in the pooled output of the final text encoder + pooled_prompt_embeds = prompt_embeds[0] + prompt_embeds = prompt_embeds.hidden_states[-2] + + prompt_embeds_list.append(prompt_embeds) + + prompt_embeds = torch.concat(prompt_embeds_list, dim=-1) + + # get unconditional embeddings for classifier free guidance + zero_out_negative_prompt = negative_prompt is None and self.config.force_zeros_for_empty_prompt + if do_classifier_free_guidance and negative_prompt_embeds is None and zero_out_negative_prompt: + negative_prompt_embeds = torch.zeros_like(prompt_embeds) + negative_pooled_prompt_embeds = torch.zeros_like(pooled_prompt_embeds) + elif do_classifier_free_guidance and negative_prompt_embeds is None: + negative_prompt = negative_prompt or "" + negative_prompt_2 = negative_prompt_2 or negative_prompt + + uncond_tokens: List[str] + if prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt, negative_prompt_2] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = [negative_prompt, negative_prompt_2] + + negative_prompt_embeds_list = [] + for negative_prompt, tokenizer, text_encoder in zip(uncond_tokens, tokenizers, text_encoders): + if isinstance(self, TextualInversionLoaderMixin): + negative_prompt = self.maybe_convert_prompt(negative_prompt, tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = tokenizer( + negative_prompt, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + negative_prompt_embeds = text_encoder( + uncond_input.input_ids.to(device), + output_hidden_states=True, + ) + # We are only ALWAYS interested in the pooled output of the final text encoder + negative_pooled_prompt_embeds = negative_prompt_embeds[0] + negative_prompt_embeds = negative_prompt_embeds.hidden_states[-2] + + negative_prompt_embeds_list.append(negative_prompt_embeds) + + negative_prompt_embeds = torch.concat(negative_prompt_embeds_list, dim=-1) + + prompt_embeds = prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device) + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device) + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + pooled_prompt_embeds = pooled_prompt_embeds.repeat(1, num_images_per_prompt).view( + bs_embed * num_images_per_prompt, -1 + ) + if do_classifier_free_guidance: + negative_pooled_prompt_embeds = negative_pooled_prompt_embeds.repeat(1, num_images_per_prompt).view( + bs_embed * num_images_per_prompt, -1 + ) + + return prompt_embeds, negative_prompt_embeds, pooled_prompt_embeds, negative_pooled_prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def check_inputs( + self, + prompt, + prompt_2, + height, + width, + callback_steps, + negative_prompt=None, + negative_prompt_2=None, + prompt_embeds=None, + negative_prompt_embeds=None, + pooled_prompt_embeds=None, + negative_pooled_prompt_embeds=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if (callback_steps is None) or ( + callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0) + ): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt_2 is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt_2`: {prompt_2} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + elif prompt_2 is not None and (not isinstance(prompt_2, str) and not isinstance(prompt_2, list)): + raise ValueError(f"`prompt_2` has to be of type `str` or `list` but is {type(prompt_2)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + elif negative_prompt_2 is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt_2`: {negative_prompt_2} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + if prompt_embeds is not None and pooled_prompt_embeds is None: + raise ValueError( + "If `prompt_embeds` are provided, `pooled_prompt_embeds` also have to be passed. Make sure to generate `pooled_prompt_embeds` from the same text encoder that was used to generate `prompt_embeds`." + ) + + if negative_prompt_embeds is not None and negative_pooled_prompt_embeds is None: + raise ValueError( + "If `negative_prompt_embeds` are provided, `negative_pooled_prompt_embeds` also have to be passed. Make sure to generate `negative_pooled_prompt_embeds` from the same text encoder that was used to generate `negative_prompt_embeds`." + ) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents + def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None): + shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + def _get_add_time_ids(self, original_size, crops_coords_top_left, target_size, dtype): + add_time_ids = list(original_size + crops_coords_top_left + target_size) + + passed_add_embed_dim = ( + self.unet.config.addition_time_embed_dim * len(add_time_ids) + self.text_encoder_2.config.projection_dim + ) + expected_add_embed_dim = self.unet.add_embedding.linear_1.in_features + + if expected_add_embed_dim != passed_add_embed_dim: + raise ValueError( + f"Model expects an added time embedding vector of length {expected_add_embed_dim}, but a vector of {passed_add_embed_dim} was created. The model has an incorrect config. Please check `unet.config.time_embedding_type` and `text_encoder_2.config.projection_dim`." + ) + + add_time_ids = torch.tensor([add_time_ids], dtype=dtype) + return add_time_ids + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_upscale.StableDiffusionUpscalePipeline.upcast_vae + def upcast_vae(self): + dtype = self.vae.dtype + self.vae.to(dtype=torch.float32) + use_torch_2_0_or_xformers = isinstance( + self.vae.decoder.mid_block.attentions[0].processor, + ( + AttnProcessor2_0, + XFormersAttnProcessor, + LoRAXFormersAttnProcessor, + LoRAAttnProcessor2_0, + ), + ) + # if xformers or torch_2_0 is used attention block does not need + # to be in float32 which can save lots of memory + if use_torch_2_0_or_xformers: + self.vae.post_quant_conv.to(dtype) + self.vae.decoder.conv_in.to(dtype) + self.vae.decoder.mid_block.to(dtype) + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + prompt_2: Optional[Union[str, List[str]]] = None, + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 50, + denoising_end: Optional[float] = None, + guidance_scale: float = 5.0, + negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt_2: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + guidance_rescale: float = 0.0, + original_size: Optional[Tuple[int, int]] = None, + crops_coords_top_left: Tuple[int, int] = (0, 0), + target_size: Optional[Tuple[int, int]] = None, + resolutions_list: Optional[Union[int, List[int]]] = None, + restart_steps: Optional[Union[int, List[int]]] = None, + cosine_scale: float = 2.0, + dilate_tau: int = 35, + ): + r""" + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. + instead. + prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is + used in both text-encoders + height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + denoising_end (`float`, *optional*): + When specified, determines the fraction (between 0.0 and 1.0) of the total denoising process to be + completed before it is intentionally prematurely terminated. As a result, the returned sample will + still retain a substantial amount of noise as determined by the discrete timesteps selected by the + scheduler. The denoising_end parameter should ideally be utilized when this pipeline forms a part of a + "Mixture of Denoisers" multi-pipeline setup, as elaborated in [**Refining the Image + Output**](https://huggingface.co/docs/diffusers/api/pipelines/stable_diffusion/stable_diffusion_xl#refining-the-image-output) + guidance_scale (`float`, *optional*, defaults to 5.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + negative_prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and + `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to + [`schedulers.DDIMScheduler`], will be ignored for others. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. + If not provided, pooled text embeddings will be generated from `prompt` input argument. + negative_pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` + input argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] instead + of a plain tuple. + callback (`Callable`, *optional*): + A function that will be called every `callback_steps` steps during inference. The function will be + called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function will be called. If not specified, the callback will be + called at every step. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + guidance_rescale (`float`, *optional*, defaults to 0.7): + Guidance rescale factor proposed by [Common Diffusion Noise Schedules and Sample Steps are + Flawed](https://arxiv.org/pdf/2305.08891.pdf) `guidance_scale` is defined as `φ` in equation 16. of + [Common Diffusion Noise Schedules and Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). + Guidance rescale factor should fix overexposure when using zero terminal SNR. + original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled. + `original_size` defaults to `(width, height)` if not specified. Part of SDXL's micro-conditioning as + explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position + `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting + `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + For most cases, `target_size` should be set to the desired height and width of the generated image. If + not specified it will default to `(width, height)`. Part of SDXL's micro-conditioning as explained in + section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + + Examples: + + Returns: + [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] or `tuple`: + [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] if `return_dict` is True, otherwise a + `tuple`. When returning a tuple, the first element is a list with the generated images. + """ + + # 0. Default height and width to unet + if resolutions_list: + height, width = resolutions_list[0] + target_sizes = resolutions_list[1:] + if not restart_steps: + restart_steps = [15] * len(target_sizes) + else: + height = height or self.default_sample_size * self.vae_scale_factor + width = width or self.default_sample_size * self.vae_scale_factor + + original_size = original_size or (height, width) + target_size = target_size or (height, width) + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + prompt_2, + height, + width, + callback_steps, + negative_prompt, + negative_prompt_2, + prompt_embeds, + negative_prompt_embeds, + pooled_prompt_embeds, + negative_pooled_prompt_embeds, + ) + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Encode input prompt + text_encoder_lora_scale = ( + cross_attention_kwargs.get("scale", None) if cross_attention_kwargs is not None else None + ) + ( + prompt_embeds, + negative_prompt_embeds, + pooled_prompt_embeds, + negative_pooled_prompt_embeds, + ) = self.encode_prompt( + prompt=prompt, + prompt_2=prompt_2, + device=device, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=do_classifier_free_guidance, + negative_prompt=negative_prompt, + negative_prompt_2=negative_prompt_2, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + pooled_prompt_embeds=pooled_prompt_embeds, + negative_pooled_prompt_embeds=negative_pooled_prompt_embeds, + lora_scale=text_encoder_lora_scale, + ) + + # 4. Prepare timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + + timesteps = self.scheduler.timesteps + + # 5. Prepare latent variables + num_channels_latents = self.unet.config.in_channels + latents = self.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + ) + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 7. Prepare added time ids & embeddings + add_text_embeds = pooled_prompt_embeds + add_time_ids = self._get_add_time_ids( + original_size, crops_coords_top_left, target_size, dtype=prompt_embeds.dtype + ) + + if do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0) + add_text_embeds = torch.cat([negative_pooled_prompt_embeds, add_text_embeds], dim=0) + add_time_ids = torch.cat([add_time_ids, add_time_ids], dim=0) + + prompt_embeds = prompt_embeds.to(device) + add_text_embeds = add_text_embeds.to(device) + add_time_ids = add_time_ids.to(device).repeat(batch_size * num_images_per_prompt, 1) + + # 8. Denoising loop + num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0) + + # 9.1 Apply denoising_end + if denoising_end is not None and type(denoising_end) == float and denoising_end > 0 and denoising_end < 1: + discrete_timestep_cutoff = int( + round( + self.scheduler.config.num_train_timesteps + - (denoising_end * self.scheduler.config.num_train_timesteps) + ) + ) + num_inference_steps = len(list(filter(lambda ts: ts >= discrete_timestep_cutoff, timesteps))) + timesteps = timesteps[:num_inference_steps] + + results_list = [] + + for block in self.unet.down_blocks + [self.unet.mid_block] + self.unet.up_blocks: + for module in block.modules(): + if isinstance(module, BasicTransformerBlock): + module.forward = ori_forward.__get__(module, BasicTransformerBlock) + + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # predict the noise residual + added_cond_kwargs = {"text_embeds": add_text_embeds, "time_ids": add_time_ids} + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + cross_attention_kwargs=cross_attention_kwargs, + added_cond_kwargs=added_cond_kwargs, + return_dict=False, + )[0] + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + if do_classifier_free_guidance and guidance_rescale > 0.0: + # Based on 3.4. in https://arxiv.org/pdf/2305.08891.pdf + noise_pred = rescale_noise_cfg(noise_pred, noise_pred_text, guidance_rescale=guidance_rescale) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + callback(i, t, latents) + results_list.append(latents) + + for restart_index, target_size in enumerate(target_sizes): + restart_step = restart_steps[restart_index] + target_size_ = [target_size[0]//8, target_size[1]//8] + + for block in self.unet.down_blocks + [self.unet.mid_block] + self.unet.up_blocks: + for module in block.modules(): + if isinstance(module, BasicTransformerBlock): + module.forward = scale_forward.__get__(module, BasicTransformerBlock) + module.current_hw = target_size + + needs_upcasting = self.vae.dtype == torch.float16 and self.vae.config.force_upcast + if needs_upcasting: + self.upcast_vae() + latents = latents.to(next(iter(self.vae.post_quant_conv.parameters())).dtype) + + latents = latents / self.vae.config.scaling_factor + image = self.vae.decode(latents, return_dict=False)[0] + image = torch.nn.functional.interpolate( + image, + size=target_size, + mode='bicubic', + ) + latents = self.vae.encode(image).latent_dist.sample().to(self.vae.dtype) + latents = latents * self.vae.config.scaling_factor + + noise_latents = [] + noise = torch.randn_like(latents) + for timestep in self.scheduler.timesteps: + noise_latent = self.scheduler.add_noise(latents, noise, timestep.unsqueeze(0)) + noise_latents.append(noise_latent) + latents = noise_latents[restart_step] + + self.scheduler._step_index = 0 + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + + if i < restart_step: + self.scheduler._step_index += 1 + progress_bar.update() + continue + + cosine_factor = 0.5 * (1 + torch.cos(torch.pi * (self.scheduler.config.num_train_timesteps - t) / self.scheduler.config.num_train_timesteps)).cpu() + c1 = cosine_factor ** cosine_scale + latents = latents * (1 - c1) + noise_latents[i] * c1 + + dilate_coef=target_size[1]//1024 + + dilate_layers = [ + # "down_blocks.1.resnets.0.conv1", + # "down_blocks.1.resnets.0.conv2", + # "down_blocks.1.resnets.1.conv1", + # "down_blocks.1.resnets.1.conv2", + "down_blocks.1.downsamplers.0.conv", + "down_blocks.2.resnets.0.conv1", + "down_blocks.2.resnets.0.conv2", + "down_blocks.2.resnets.1.conv1", + "down_blocks.2.resnets.1.conv2", + # "up_blocks.0.resnets.0.conv1", + # "up_blocks.0.resnets.0.conv2", + # "up_blocks.0.resnets.1.conv1", + # "up_blocks.0.resnets.1.conv2", + # "up_blocks.0.resnets.2.conv1", + # "up_blocks.0.resnets.2.conv2", + # "up_blocks.0.upsamplers.0.conv", + # "up_blocks.1.resnets.0.conv1", + # "up_blocks.1.resnets.0.conv2", + # "up_blocks.1.resnets.1.conv1", + # "up_blocks.1.resnets.1.conv2", + # "up_blocks.1.resnets.2.conv1", + # "up_blocks.1.resnets.2.conv2", + # "up_blocks.1.upsamplers.0.conv", + # "up_blocks.2.resnets.0.conv1", + # "up_blocks.2.resnets.0.conv2", + # "up_blocks.2.resnets.1.conv1", + # "up_blocks.2.resnets.1.conv2", + # "up_blocks.2.resnets.2.conv1", + # "up_blocks.2.resnets.2.conv2", + "mid_block.resnets.0.conv1", + "mid_block.resnets.0.conv2", + "mid_block.resnets.1.conv1", + "mid_block.resnets.1.conv2" + ] + + for name, module in self.unet.named_modules(): + if name in dilate_layers: + if i < dilate_tau: + module.dilation = (dilate_coef, dilate_coef) + module.padding = (dilate_coef, dilate_coef) + else: + module.dilation = (1, 1) + module.padding = (1, 1) + + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + + # predict the noise residual + added_cond_kwargs = {"text_embeds": add_text_embeds, "time_ids": add_time_ids} + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + cross_attention_kwargs=cross_attention_kwargs, + added_cond_kwargs=added_cond_kwargs, + return_dict=False, + )[0] + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + if do_classifier_free_guidance and guidance_rescale > 0.0: + # Based on 3.4. in https://arxiv.org/pdf/2305.08891.pdf + noise_pred = rescale_noise_cfg(noise_pred, noise_pred_text, guidance_rescale=guidance_rescale) + + # compute the previous noisy sample x_t -> x_t-1 + latents_dtype = latents.dtype + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + if latents.dtype != latents_dtype: + if torch.backends.mps.is_available(): + # some platforms (eg. apple mps) misbehave due to a pytorch bug: https://github.com/pytorch/pytorch/pull/99272 + latents = latents.to(latents_dtype) + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + callback(i, t, latents) + + for name, module in self.unet.named_modules(): + # if ('.conv' in name) and ('.conv_' not in name): + if name in dilate_layers: + module.dilation = (1, 1) + module.padding = (1, 1) + + results_list.append(latents) + + """ + final_results = [] + for latents in results_list: + # make sure the VAE is in float32 mode, as it overflows in float16 + if self.vae.dtype == torch.float16 and self.vae.config.force_upcast: + self.upcast_vae() + latents = latents.to(next(iter(self.vae.post_quant_conv.parameters())).dtype) + + if not output_type == "latent": + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0] + else: + image = latents + return StableDiffusionXLPipelineOutput(images=image) + + image = self.image_processor.postprocess(image, output_type=output_type) + + if not return_dict: + final_results += [(image,)] + else: + final_results += [StableDiffusionXLPipelineOutput(images=image)] + + # Offload last model to CPU + if hasattr(self, "final_offload_hook") and self.final_offload_hook is not None: + self.final_offload_hook.offload() + + return final_results + """ + return StableDiffusionXLPipelineOutput(images=results_list) + + # Overrride to properly handle the loading and unloading of the additional text encoder. + def load_lora_weights(self, pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]], **kwargs): + # We could have accessed the unet config from `lora_state_dict()` too. We pass + # it here explicitly to be able to tell that it's coming from an SDXL + # pipeline. + state_dict, network_alphas = self.lora_state_dict( + pretrained_model_name_or_path_or_dict, + unet_config=self.unet.config, + **kwargs, + ) + self.load_lora_into_unet(state_dict, network_alphas=network_alphas, unet=self.unet) + + text_encoder_state_dict = {k: v for k, v in state_dict.items() if "text_encoder." in k} + if len(text_encoder_state_dict) > 0: + self.load_lora_into_text_encoder( + text_encoder_state_dict, + network_alphas=network_alphas, + text_encoder=self.text_encoder, + prefix="text_encoder", + lora_scale=self.lora_scale, + ) + + text_encoder_2_state_dict = {k: v for k, v in state_dict.items() if "text_encoder_2." in k} + if len(text_encoder_2_state_dict) > 0: + self.load_lora_into_text_encoder( + text_encoder_2_state_dict, + network_alphas=network_alphas, + text_encoder=self.text_encoder_2, + prefix="text_encoder_2", + lora_scale=self.lora_scale, + ) + + @classmethod + def save_lora_weights( + self, + save_directory: Union[str, os.PathLike], + unet_lora_layers: Dict[str, Union[torch.nn.Module, torch.Tensor]] = None, + text_encoder_lora_layers: Dict[str, Union[torch.nn.Module, torch.Tensor]] = None, + text_encoder_2_lora_layers: Dict[str, Union[torch.nn.Module, torch.Tensor]] = None, + is_main_process: bool = True, + weight_name: str = None, + save_function: Callable = None, + safe_serialization: bool = True, + ): + state_dict = {} + + def pack_weights(layers, prefix): + layers_weights = layers.state_dict() if isinstance(layers, torch.nn.Module) else layers + layers_state_dict = {f"{prefix}.{module_name}": param for module_name, param in layers_weights.items()} + return layers_state_dict + + state_dict.update(pack_weights(unet_lora_layers, "unet")) + + if text_encoder_lora_layers and text_encoder_2_lora_layers: + state_dict.update(pack_weights(text_encoder_lora_layers, "text_encoder")) + state_dict.update(pack_weights(text_encoder_2_lora_layers, "text_encoder_2")) + + self.write_lora_layers( + state_dict=state_dict, + save_directory=save_directory, + is_main_process=is_main_process, + weight_name=weight_name, + save_function=save_function, + safe_serialization=safe_serialization, + ) + + def _remove_text_encoder_monkey_patch(self): + self._remove_text_encoder_monkey_patch_classmethod(self.text_encoder) + self._remove_text_encoder_monkey_patch_classmethod(self.text_encoder_2) diff --git a/modules/freescale/freescale_pipeline_img2img.py b/modules/freescale/freescale_pipeline_img2img.py new file mode 100644 index 000000000..df4c3f0c1 --- /dev/null +++ b/modules/freescale/freescale_pipeline_img2img.py @@ -0,0 +1,1245 @@ +from inspect import isfunction +from functools import partial +from typing import Any, Callable, Dict, List, Optional, Tuple, Union +import inspect +import os +import random + +from PIL import Image +import numpy as np +import torch +import torch.nn.functional as F +from einops import rearrange +from transformers import CLIPTextModel, CLIPTextModelWithProjection, CLIPTokenizer +import torchvision.transforms as transforms + +from diffusers.image_processor import VaeImageProcessor +from diffusers.loaders import FromSingleFileMixin, LoraLoaderMixin, TextualInversionLoaderMixin +from diffusers.models import AutoencoderKL, UNet2DConditionModel +from diffusers.models.attention_processor import AttnProcessor2_0, LoRAAttnProcessor2_0, LoRAXFormersAttnProcessor, XFormersAttnProcessor +from diffusers.schedulers import KarrasDiffusionSchedulers +from diffusers.utils.torch_utils import randn_tensor +from diffusers.utils import is_accelerate_available, is_accelerate_version, logging, replace_example_docstring +from diffusers.pipelines.pipeline_utils import DiffusionPipeline +from diffusers.pipelines.stable_diffusion_xl.pipeline_output import StableDiffusionXLPipelineOutput +from diffusers.models.attention import BasicTransformerBlock + +from .scale_attention import ori_forward, scale_forward + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> from diffusers import StableDiffusionXLPipeline + + >>> pipe = StableDiffusionXLPipeline.from_pretrained( + ... "stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16 + ... ) + >>> pipe = pipe.to("cuda") + + >>> prompt = "a photo of an astronaut riding a horse on mars" + >>> image = pipe(prompt).images[0] + ``` +""" + +def process_image_to_tensor(image): + image = image.convert("RGB") + # image = Image.open(image_path).convert("RGB") + transform = transforms.Compose( + [ + # transforms.Resize((1024, 1024)), + transforms.ToTensor(), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]), + ] + ) + image_tensor = transform(image) + return image_tensor + +def process_image_to_bitensor(image): + # image = Image.open(image_path).convert("L") + image = image.convert("L") + transform = transforms.ToTensor() + image_tensor = transform(image) + binary_tensor = torch.where(image_tensor != 0, torch.tensor(1.0), torch.tensor(0.0)) + return binary_tensor + +def default(val, d): + if exists(val): + return val + return d() if isfunction(d) else d + +def exists(val): + return val is not None + +def extract_into_tensor(a, t, x_shape): + b, *_ = t.shape + out = a.gather(-1, t) + return out.reshape(b, *((1,) * (len(x_shape) - 1))) + +def make_beta_schedule(schedule, n_timestep, linear_start=1e-4, linear_end=2e-2, cosine_s=8e-3): + if schedule == "linear": + betas = ( + torch.linspace(linear_start ** 0.5, linear_end ** 0.5, n_timestep, dtype=torch.float64) ** 2 + ) + elif schedule == "cosine": + timesteps = ( + torch.arange(n_timestep + 1, dtype=torch.float64) / n_timestep + cosine_s + ) + alphas = timesteps / (1 + cosine_s) * np.pi / 2 + alphas = torch.cos(alphas).pow(2) + alphas = alphas / alphas[0] + betas = 1 - alphas[1:] / alphas[:-1] + betas = np.clip(betas, a_min=0, a_max=0.999) + elif schedule == "sqrt_linear": + betas = torch.linspace(linear_start, linear_end, n_timestep, dtype=torch.float64) + elif schedule == "sqrt": + betas = torch.linspace(linear_start, linear_end, n_timestep, dtype=torch.float64) ** 0.5 + else: + raise ValueError(f"schedule '{schedule}' unknown.") + return betas.numpy() + +to_torch = partial(torch.tensor, dtype=torch.float16) +betas = make_beta_schedule("linear", 1000, linear_start=0.00085, linear_end=0.012) +alphas = 1. - betas +alphas_cumprod = np.cumprod(alphas, axis=0) +sqrt_alphas_cumprod = to_torch(np.sqrt(alphas_cumprod)) +sqrt_one_minus_alphas_cumprod = to_torch(np.sqrt(1. - alphas_cumprod)) + +def q_sample(x_start, t, init_noise_sigma = 1.0, noise=None, device=None): + noise = default(noise, lambda: torch.randn_like(x_start)).to(device) * init_noise_sigma + return (extract_into_tensor(sqrt_alphas_cumprod.to(device), t, x_start.shape) * x_start + + extract_into_tensor(sqrt_one_minus_alphas_cumprod.to(device), t, x_start.shape) * noise) + +def get_views(height, width, h_window_size=128, w_window_size=128, h_window_stride=64, w_window_stride=64, vae_scale_factor=8): + height //= vae_scale_factor + width //= vae_scale_factor + num_blocks_height = int((height - h_window_size) / h_window_stride - 1e-6) + 2 if height > h_window_size else 1 + num_blocks_width = int((width - w_window_size) / w_window_stride - 1e-6) + 2 if width > w_window_size else 1 + total_num_blocks = int(num_blocks_height * num_blocks_width) + views = [] + for i in range(total_num_blocks): + h_start = int((i // num_blocks_width) * h_window_stride) + h_end = h_start + h_window_size + w_start = int((i % num_blocks_width) * w_window_stride) + w_end = w_start + w_window_size + + if h_end > height: + h_start = int(h_start + height - h_end) + h_end = int(height) + if w_end > width: + w_start = int(w_start + width - w_end) + w_end = int(width) + if h_start < 0: + h_end = int(h_end - h_start) + h_start = 0 + if w_start < 0: + w_end = int(w_end - w_start) + w_start = 0 + + random_jitter = True + if random_jitter: + h_jitter_range = (h_window_size - h_window_stride) // 4 + w_jitter_range = (w_window_size - w_window_stride) // 4 + h_jitter = 0 + w_jitter = 0 + + if (w_start != 0) and (w_end != width): + w_jitter = random.randint(-w_jitter_range, w_jitter_range) + elif (w_start == 0) and (w_end != width): + w_jitter = random.randint(-w_jitter_range, 0) + elif (w_start != 0) and (w_end == width): + w_jitter = random.randint(0, w_jitter_range) + if (h_start != 0) and (h_end != height): + h_jitter = random.randint(-h_jitter_range, h_jitter_range) + elif (h_start == 0) and (h_end != height): + h_jitter = random.randint(-h_jitter_range, 0) + elif (h_start != 0) and (h_end == height): + h_jitter = random.randint(0, h_jitter_range) + h_start += (h_jitter + h_jitter_range) + h_end += (h_jitter + h_jitter_range) + w_start += (w_jitter + w_jitter_range) + w_end += (w_jitter + w_jitter_range) + + views.append((h_start, h_end, w_start, w_end)) + return views + +def gaussian_kernel(kernel_size=3, sigma=1.0, channels=3): + x_coord = torch.arange(kernel_size) + gaussian_1d = torch.exp(-(x_coord - (kernel_size - 1) / 2) ** 2 / (2 * sigma ** 2)) + gaussian_1d = gaussian_1d / gaussian_1d.sum() + gaussian_2d = gaussian_1d[:, None] * gaussian_1d[None, :] + kernel = gaussian_2d[None, None, :, :].repeat(channels, 1, 1, 1) + + return kernel + +def gaussian_filter(latents, kernel_size=3, sigma=1.0): + channels = latents.shape[1] + kernel = gaussian_kernel(kernel_size, sigma, channels).to(latents.device, latents.dtype) + if len(latents.shape) == 5: + b = latents.shape[0] + latents = rearrange(latents, 'b c t i j -> (b t) c i j') + blurred_latents = F.conv2d(latents, kernel, padding=kernel_size//2, groups=channels) + blurred_latents = rearrange(blurred_latents, '(b t) c i j -> b c t i j', b=b) + else: + blurred_latents = F.conv2d(latents, kernel, padding=kernel_size//2, groups=channels) + + return blurred_latents + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.rescale_noise_cfg +def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0): + """ + Rescale `noise_cfg` according to `guidance_rescale`. Based on findings of [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). See Section 3.4 + """ + std_text = noise_pred_text.std(dim=list(range(1, noise_pred_text.ndim)), keepdim=True) + std_cfg = noise_cfg.std(dim=list(range(1, noise_cfg.ndim)), keepdim=True) + # rescale the results from guidance (fixes overexposure) + noise_pred_rescaled = noise_cfg * (std_text / std_cfg) + # mix with the original results from guidance by factor guidance_rescale to avoid "plain looking" images + noise_cfg = guidance_rescale * noise_pred_rescaled + (1 - guidance_rescale) * noise_cfg + return noise_cfg + + +class StableDiffusionXLFreeScaleImg2Img(DiffusionPipeline, FromSingleFileMixin, LoraLoaderMixin): + r""" + Pipeline for text-to-image generation using Stable Diffusion XL. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + In addition the pipeline inherits the following loading methods: + - *LoRA*: [`StableDiffusionXLPipeline.load_lora_weights`] + - *Ckpt*: [`loaders.FromSingleFileMixin.from_single_file`] + + as well as the following saving methods: + - *LoRA*: [`loaders.StableDiffusionXLPipeline.save_lora_weights`] + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. + text_encoder ([`CLIPTextModel`]): + Frozen text-encoder. Stable Diffusion XL uses the text portion of + [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically + the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant. + text_encoder_2 ([` CLIPTextModelWithProjection`]): + Second frozen text-encoder. Stable Diffusion XL uses the text and pool portion of + [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModelWithProjection), + specifically the + [laion/CLIP-ViT-bigG-14-laion2B-39B-b160k](https://huggingface.co/laion/CLIP-ViT-bigG-14-laion2B-39B-b160k) + variant. + tokenizer (`CLIPTokenizer`): + Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). + tokenizer_2 (`CLIPTokenizer`): + Second Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). + unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + """ + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + text_encoder_2: CLIPTextModelWithProjection, + tokenizer: CLIPTokenizer, + tokenizer_2: CLIPTokenizer, + unet: UNet2DConditionModel, + scheduler: KarrasDiffusionSchedulers, + force_zeros_for_empty_prompt: bool = True, + add_watermarker: Optional[bool] = None, + ): + super().__init__() + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + text_encoder_2=text_encoder_2, + tokenizer=tokenizer, + tokenizer_2=tokenizer_2, + unet=unet, + scheduler=scheduler, + ) + self.register_to_config(force_zeros_for_empty_prompt=force_zeros_for_empty_prompt) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + self.default_sample_size = self.unet.config.sample_size + + self.vae.enable_tiling() + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.enable_vae_slicing + def enable_vae_slicing(self): + r""" + Enable sliced VAE decoding. When this option is enabled, the VAE will split the input tensor in slices to + compute decoding in several steps. This is useful to save some memory and allow larger batch sizes. + """ + self.vae.enable_slicing() + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.disable_vae_slicing + def disable_vae_slicing(self): + r""" + Disable sliced VAE decoding. If `enable_vae_slicing` was previously enabled, this method will go back to + computing decoding in one step. + """ + self.vae.disable_slicing() + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.enable_vae_tiling + def enable_vae_tiling(self): + r""" + Enable tiled VAE decoding. When this option is enabled, the VAE will split the input tensor into tiles to + compute decoding and encoding in several steps. This is useful for saving a large amount of memory and to allow + processing larger images. + """ + self.vae.enable_tiling() + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.disable_vae_tiling + def disable_vae_tiling(self): + r""" + Disable tiled VAE decoding. If `enable_vae_tiling` was previously enabled, this method will go back to + computing decoding in one step. + """ + self.vae.disable_tiling() + + def enable_model_cpu_offload(self, gpu_id=0): + r""" + Offloads all models to CPU using accelerate, reducing memory usage with a low impact on performance. Compared + to `enable_sequential_cpu_offload`, this method moves one whole model at a time to the GPU when its `forward` + method is called, and the model remains in GPU until the next model runs. Memory savings are lower than with + `enable_sequential_cpu_offload`, but performance is much better due to the iterative execution of the `unet`. + """ + if is_accelerate_available() and is_accelerate_version(">=", "0.17.0.dev0"): + from accelerate import cpu_offload_with_hook + else: + raise ImportError("`enable_model_cpu_offload` requires `accelerate v0.17.0` or higher.") + + device = torch.device(f"cuda:{gpu_id}") + + if self.device.type != "cpu": + self.to("cpu", silence_dtype_warnings=True) + torch.cuda.empty_cache() # otherwise we don't see the memory savings (but they probably exist) + + model_sequence = ( + [self.text_encoder, self.text_encoder_2] if self.text_encoder is not None else [self.text_encoder_2] + ) + model_sequence.extend([self.unet, self.vae]) + + hook = None + for cpu_offloaded_model in model_sequence: + _, hook = cpu_offload_with_hook(cpu_offloaded_model, device, prev_module_hook=hook) + + # We'll offload the last model manually. + self.final_offload_hook = hook + + def encode_prompt( + self, + prompt: str, + prompt_2: Optional[str] = None, + device: Optional[torch.device] = None, + num_images_per_prompt: int = 1, + do_classifier_free_guidance: bool = True, + negative_prompt: Optional[str] = None, + negative_prompt_2: Optional[str] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is + used in both text-encoders + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + negative_prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and + `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. + If not provided, pooled text embeddings will be generated from `prompt` input argument. + negative_pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` + input argument. + lora_scale (`float`, *optional*): + A lora scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + """ + device = device or self._execution_device + + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, LoraLoaderMixin): + self._lora_scale = lora_scale + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + # Define tokenizers and text encoders + tokenizers = [self.tokenizer, self.tokenizer_2] if self.tokenizer is not None else [self.tokenizer_2] + text_encoders = ( + [self.text_encoder, self.text_encoder_2] if self.text_encoder is not None else [self.text_encoder_2] + ) + + if prompt_embeds is None: + prompt_2 = prompt_2 or prompt + # textual inversion: procecss multi-vector tokens if necessary + prompt_embeds_list = [] + prompts = [prompt, prompt_2] + for prompt, tokenizer, text_encoder in zip(prompts, tokenizers, text_encoders): + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, tokenizer) + + text_inputs = tokenizer( + prompt, + padding="max_length", + max_length=tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + + text_input_ids = text_inputs.input_ids + untruncated_ids = tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = tokenizer.batch_decode(untruncated_ids[:, tokenizer.model_max_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {tokenizer.model_max_length} tokens: {removed_text}" + ) + + prompt_embeds = text_encoder( + text_input_ids.to(device), + output_hidden_states=True, + ) + + # We are only ALWAYS interested in the pooled output of the final text encoder + pooled_prompt_embeds = prompt_embeds[0] + prompt_embeds = prompt_embeds.hidden_states[-2] + + prompt_embeds_list.append(prompt_embeds) + + prompt_embeds = torch.concat(prompt_embeds_list, dim=-1) + + # get unconditional embeddings for classifier free guidance + zero_out_negative_prompt = negative_prompt is None and self.config.force_zeros_for_empty_prompt + if do_classifier_free_guidance and negative_prompt_embeds is None and zero_out_negative_prompt: + negative_prompt_embeds = torch.zeros_like(prompt_embeds) + negative_pooled_prompt_embeds = torch.zeros_like(pooled_prompt_embeds) + elif do_classifier_free_guidance and negative_prompt_embeds is None: + negative_prompt = negative_prompt or "" + negative_prompt_2 = negative_prompt_2 or negative_prompt + + uncond_tokens: List[str] + if prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt, negative_prompt_2] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = [negative_prompt, negative_prompt_2] + + negative_prompt_embeds_list = [] + for negative_prompt, tokenizer, text_encoder in zip(uncond_tokens, tokenizers, text_encoders): + if isinstance(self, TextualInversionLoaderMixin): + negative_prompt = self.maybe_convert_prompt(negative_prompt, tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = tokenizer( + negative_prompt, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + negative_prompt_embeds = text_encoder( + uncond_input.input_ids.to(device), + output_hidden_states=True, + ) + # We are only ALWAYS interested in the pooled output of the final text encoder + negative_pooled_prompt_embeds = negative_prompt_embeds[0] + negative_prompt_embeds = negative_prompt_embeds.hidden_states[-2] + + negative_prompt_embeds_list.append(negative_prompt_embeds) + + negative_prompt_embeds = torch.concat(negative_prompt_embeds_list, dim=-1) + + prompt_embeds = prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device) + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device) + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + pooled_prompt_embeds = pooled_prompt_embeds.repeat(1, num_images_per_prompt).view( + bs_embed * num_images_per_prompt, -1 + ) + if do_classifier_free_guidance: + negative_pooled_prompt_embeds = negative_pooled_prompt_embeds.repeat(1, num_images_per_prompt).view( + bs_embed * num_images_per_prompt, -1 + ) + + return prompt_embeds, negative_prompt_embeds, pooled_prompt_embeds, negative_pooled_prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def check_inputs( + self, + prompt, + prompt_2, + height, + width, + callback_steps, + negative_prompt=None, + negative_prompt_2=None, + prompt_embeds=None, + negative_prompt_embeds=None, + pooled_prompt_embeds=None, + negative_pooled_prompt_embeds=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if (callback_steps is None) or ( + callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0) + ): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt_2 is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt_2`: {prompt_2} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + elif prompt_2 is not None and (not isinstance(prompt_2, str) and not isinstance(prompt_2, list)): + raise ValueError(f"`prompt_2` has to be of type `str` or `list` but is {type(prompt_2)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + elif negative_prompt_2 is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt_2`: {negative_prompt_2} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + if prompt_embeds is not None and pooled_prompt_embeds is None: + raise ValueError( + "If `prompt_embeds` are provided, `pooled_prompt_embeds` also have to be passed. Make sure to generate `pooled_prompt_embeds` from the same text encoder that was used to generate `prompt_embeds`." + ) + + if negative_prompt_embeds is not None and negative_pooled_prompt_embeds is None: + raise ValueError( + "If `negative_prompt_embeds` are provided, `negative_pooled_prompt_embeds` also have to be passed. Make sure to generate `negative_pooled_prompt_embeds` from the same text encoder that was used to generate `negative_prompt_embeds`." + ) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents + def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None): + shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + def _get_add_time_ids(self, original_size, crops_coords_top_left, target_size, dtype): + add_time_ids = list(original_size + crops_coords_top_left + target_size) + + passed_add_embed_dim = ( + self.unet.config.addition_time_embed_dim * len(add_time_ids) + self.text_encoder_2.config.projection_dim + ) + expected_add_embed_dim = self.unet.add_embedding.linear_1.in_features + + if expected_add_embed_dim != passed_add_embed_dim: + raise ValueError( + f"Model expects an added time embedding vector of length {expected_add_embed_dim}, but a vector of {passed_add_embed_dim} was created. The model has an incorrect config. Please check `unet.config.time_embedding_type` and `text_encoder_2.config.projection_dim`." + ) + + add_time_ids = torch.tensor([add_time_ids], dtype=dtype) + return add_time_ids + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_upscale.StableDiffusionUpscalePipeline.upcast_vae + def upcast_vae(self): + dtype = self.vae.dtype + self.vae.to(dtype=torch.float32) + use_torch_2_0_or_xformers = isinstance( + self.vae.decoder.mid_block.attentions[0].processor, + ( + AttnProcessor2_0, + XFormersAttnProcessor, + LoRAXFormersAttnProcessor, + LoRAAttnProcessor2_0, + ), + ) + # if xformers or torch_2_0 is used attention block does not need + # to be in float32 which can save lots of memory + if use_torch_2_0_or_xformers: + self.vae.post_quant_conv.to(dtype) + self.vae.decoder.conv_in.to(dtype) + self.vae.decoder.mid_block.to(dtype) + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + prompt_2: Optional[Union[str, List[str]]] = None, + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 50, + denoising_end: Optional[float] = None, + guidance_scale: float = 5.0, + negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt_2: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + guidance_rescale: float = 0.0, + original_size: Optional[Tuple[int, int]] = None, + crops_coords_top_left: Tuple[int, int] = (0, 0), + target_size: Optional[Tuple[int, int]] = None, + resolutions_list: Optional[Union[int, List[int]]] = None, + restart_steps: Optional[Union[int, List[int]]] = None, + cosine_scale: float = 2.0, + cosine_scale_bg: float = 1.0, + dilate_tau: int = 35, + img_path: Optional[str] = "", + mask_path: Optional[str] = "", + ): + r""" + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. + instead. + prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is + used in both text-encoders + height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + denoising_end (`float`, *optional*): + When specified, determines the fraction (between 0.0 and 1.0) of the total denoising process to be + completed before it is intentionally prematurely terminated. As a result, the returned sample will + still retain a substantial amount of noise as determined by the discrete timesteps selected by the + scheduler. The denoising_end parameter should ideally be utilized when this pipeline forms a part of a + "Mixture of Denoisers" multi-pipeline setup, as elaborated in [**Refining the Image + Output**](https://huggingface.co/docs/diffusers/api/pipelines/stable_diffusion/stable_diffusion_xl#refining-the-image-output) + guidance_scale (`float`, *optional*, defaults to 5.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + negative_prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and + `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to + [`schedulers.DDIMScheduler`], will be ignored for others. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. + If not provided, pooled text embeddings will be generated from `prompt` input argument. + negative_pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` + input argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] instead + of a plain tuple. + callback (`Callable`, *optional*): + A function that will be called every `callback_steps` steps during inference. The function will be + called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function will be called. If not specified, the callback will be + called at every step. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + guidance_rescale (`float`, *optional*, defaults to 0.7): + Guidance rescale factor proposed by [Common Diffusion Noise Schedules and Sample Steps are + Flawed](https://arxiv.org/pdf/2305.08891.pdf) `guidance_scale` is defined as `φ` in equation 16. of + [Common Diffusion Noise Schedules and Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). + Guidance rescale factor should fix overexposure when using zero terminal SNR. + original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled. + `original_size` defaults to `(width, height)` if not specified. Part of SDXL's micro-conditioning as + explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position + `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting + `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + For most cases, `target_size` should be set to the desired height and width of the generated image. If + not specified it will default to `(width, height)`. Part of SDXL's micro-conditioning as explained in + section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + + Examples: + + Returns: + [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] or `tuple`: + [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] if `return_dict` is True, otherwise a + `tuple`. When returning a tuple, the first element is a list with the generated images. + """ + + + # 0. Default height and width to unet + if resolutions_list: + height, width = resolutions_list[0] + target_sizes = resolutions_list[1:] + if not restart_steps: + restart_steps = [15] * len(target_sizes) + else: + height = height or self.default_sample_size * self.vae_scale_factor + width = width or self.default_sample_size * self.vae_scale_factor + + original_size = original_size or (height, width) + target_size = target_size or (height, width) + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + prompt_2, + height, + width, + callback_steps, + negative_prompt, + negative_prompt_2, + prompt_embeds, + negative_prompt_embeds, + pooled_prompt_embeds, + negative_pooled_prompt_embeds, + ) + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Encode input prompt + text_encoder_lora_scale = ( + cross_attention_kwargs.get("scale", None) if cross_attention_kwargs is not None else None + ) + ( + prompt_embeds, + negative_prompt_embeds, + pooled_prompt_embeds, + negative_pooled_prompt_embeds, + ) = self.encode_prompt( + prompt=prompt, + prompt_2=prompt_2, + device=device, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=do_classifier_free_guidance, + negative_prompt=negative_prompt, + negative_prompt_2=negative_prompt_2, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + pooled_prompt_embeds=pooled_prompt_embeds, + negative_pooled_prompt_embeds=negative_pooled_prompt_embeds, + lora_scale=text_encoder_lora_scale, + ) + + # 4. Prepare timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + + timesteps = self.scheduler.timesteps + + # 5. Prepare latent variables + num_channels_latents = self.unet.config.in_channels + latents = self.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + ) + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 7. Prepare added time ids & embeddings + add_text_embeds = pooled_prompt_embeds + add_time_ids = self._get_add_time_ids( + original_size, crops_coords_top_left, target_size, dtype=prompt_embeds.dtype + ) + + if do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0) + add_text_embeds = torch.cat([negative_pooled_prompt_embeds, add_text_embeds], dim=0) + add_time_ids = torch.cat([add_time_ids, add_time_ids], dim=0) + + prompt_embeds = prompt_embeds.to(device) + add_text_embeds = add_text_embeds.to(device) + add_time_ids = add_time_ids.to(device).repeat(batch_size * num_images_per_prompt, 1) + + # 8. Denoising loop + num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0) + + # 9.1 Apply denoising_end + if denoising_end is not None and type(denoising_end) == float and denoising_end > 0 and denoising_end < 1: + discrete_timestep_cutoff = int( + round( + self.scheduler.config.num_train_timesteps + - (denoising_end * self.scheduler.config.num_train_timesteps) + ) + ) + num_inference_steps = len(list(filter(lambda ts: ts >= discrete_timestep_cutoff, timesteps))) + timesteps = timesteps[:num_inference_steps] + + results_list = [] + + for block in self.unet.down_blocks + [self.unet.mid_block] + self.unet.up_blocks: + for module in block.modules(): + if isinstance(module, BasicTransformerBlock): + module.forward = ori_forward.__get__(module, BasicTransformerBlock) + + if img_path != '': + needs_upcasting = self.vae.dtype == torch.float16 and self.vae.config.force_upcast + if needs_upcasting: + self.upcast_vae() + latents = latents.to(next(iter(self.vae.post_quant_conv.parameters())).dtype) + input_image = process_image_to_tensor(img_path).unsqueeze(0).to(dtype=self.vae.dtype, device=device) + latents = self.vae.encode(input_image).latent_dist.sample().to(self.vae.dtype) + latents = latents * self.vae.config.scaling_factor + else: + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # predict the noise residual + added_cond_kwargs = {"text_embeds": add_text_embeds, "time_ids": add_time_ids} + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + cross_attention_kwargs=cross_attention_kwargs, + added_cond_kwargs=added_cond_kwargs, + return_dict=False, + )[0] + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + if do_classifier_free_guidance and guidance_rescale > 0.0: + # Based on 3.4. in https://arxiv.org/pdf/2305.08891.pdf + noise_pred = rescale_noise_cfg(noise_pred, noise_pred_text, guidance_rescale=guidance_rescale) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + callback(i, t, latents) + + results_list.append(latents) + + if mask_path != '': + mask = process_image_to_bitensor(mask_path).unsqueeze(0) + + for restart_index, target_size in enumerate(target_sizes): + restart_step = restart_steps[restart_index] + target_size_ = [target_size[0]//8, target_size[1]//8] + + for block in self.unet.down_blocks + [self.unet.mid_block] + self.unet.up_blocks: + for module in block.modules(): + if isinstance(module, BasicTransformerBlock): + module.forward = scale_forward.__get__(module, BasicTransformerBlock) + module.current_hw = target_size + + needs_upcasting = self.vae.dtype == torch.float16 and self.vae.config.force_upcast + if needs_upcasting: + self.upcast_vae() + latents = latents.to(next(iter(self.vae.post_quant_conv.parameters())).dtype) + + latents = latents / self.vae.config.scaling_factor + image = self.vae.decode(latents, return_dict=False)[0] + image = torch.nn.functional.interpolate( + image, + size=target_size, + mode='bicubic', + ) + latents = self.vae.encode(image).latent_dist.sample().to(self.vae.dtype) + latents = latents * self.vae.config.scaling_factor + + if mask_path != '': + mask_ = torch.nn.functional.interpolate( + mask, + size=target_size_, + mode="nearest", + ).to(device) + + noise_latents = [] + noise = torch.randn_like(latents) + for timestep in self.scheduler.timesteps: + noise_latent = self.scheduler.add_noise(latents, noise, timestep.unsqueeze(0)) + noise_latents.append(noise_latent) + latents = noise_latents[restart_step] + + self.scheduler._step_index = 0 + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + + if i < restart_step: + self.scheduler._step_index += 1 + progress_bar.update() + continue + + cosine_factor = 0.5 * (1 + torch.cos(torch.pi * (self.scheduler.config.num_train_timesteps - t) / self.scheduler.config.num_train_timesteps)).cpu() + if mask_path != '': + c1 = (cosine_factor ** (mask_ * cosine_scale + (1-mask_) * cosine_scale_bg)).to(dtype=torch.float16) + else: + c1 = cosine_factor ** cosine_scale + latents = latents * (1 - c1) + noise_latents[i] * c1 + + dilate_coef=target_size[1]//1024 + + dilate_layers = [ + # "down_blocks.1.resnets.0.conv1", + # "down_blocks.1.resnets.0.conv2", + # "down_blocks.1.resnets.1.conv1", + # "down_blocks.1.resnets.1.conv2", + "down_blocks.1.downsamplers.0.conv", + "down_blocks.2.resnets.0.conv1", + "down_blocks.2.resnets.0.conv2", + "down_blocks.2.resnets.1.conv1", + "down_blocks.2.resnets.1.conv2", + # "up_blocks.0.resnets.0.conv1", + # "up_blocks.0.resnets.0.conv2", + # "up_blocks.0.resnets.1.conv1", + # "up_blocks.0.resnets.1.conv2", + # "up_blocks.0.resnets.2.conv1", + # "up_blocks.0.resnets.2.conv2", + # "up_blocks.0.upsamplers.0.conv", + # "up_blocks.1.resnets.0.conv1", + # "up_blocks.1.resnets.0.conv2", + # "up_blocks.1.resnets.1.conv1", + # "up_blocks.1.resnets.1.conv2", + # "up_blocks.1.resnets.2.conv1", + # "up_blocks.1.resnets.2.conv2", + # "up_blocks.1.upsamplers.0.conv", + # "up_blocks.2.resnets.0.conv1", + # "up_blocks.2.resnets.0.conv2", + # "up_blocks.2.resnets.1.conv1", + # "up_blocks.2.resnets.1.conv2", + # "up_blocks.2.resnets.2.conv1", + # "up_blocks.2.resnets.2.conv2", + "mid_block.resnets.0.conv1", + "mid_block.resnets.0.conv2", + "mid_block.resnets.1.conv1", + "mid_block.resnets.1.conv2" + ] + + for name, module in self.unet.named_modules(): + if name in dilate_layers: + if i < dilate_tau: + module.dilation = (dilate_coef, dilate_coef) + module.padding = (dilate_coef, dilate_coef) + else: + module.dilation = (1, 1) + module.padding = (1, 1) + + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + + # predict the noise residual + added_cond_kwargs = {"text_embeds": add_text_embeds, "time_ids": add_time_ids} + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + cross_attention_kwargs=cross_attention_kwargs, + added_cond_kwargs=added_cond_kwargs, + return_dict=False, + )[0] + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + if do_classifier_free_guidance and guidance_rescale > 0.0: + # Based on 3.4. in https://arxiv.org/pdf/2305.08891.pdf + noise_pred = rescale_noise_cfg(noise_pred, noise_pred_text, guidance_rescale=guidance_rescale) + + # compute the previous noisy sample x_t -> x_t-1 + latents_dtype = latents.dtype + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + if latents.dtype != latents_dtype: + if torch.backends.mps.is_available(): + # some platforms (eg. apple mps) misbehave due to a pytorch bug: https://github.com/pytorch/pytorch/pull/99272 + latents = latents.to(latents_dtype) + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + callback(i, t, latents) + + for name, module in self.unet.named_modules(): + # if ('.conv' in name) and ('.conv_' not in name): + if name in dilate_layers: + module.dilation = (1, 1) + module.padding = (1, 1) + + results_list.append(latents) + + """ + final_results = [] + for latents in results_list: + # make sure the VAE is in float32 mode, as it overflows in float16 + if self.vae.dtype == torch.float16 and self.vae.config.force_upcast: + self.upcast_vae() + latents = latents.to(next(iter(self.vae.post_quant_conv.parameters())).dtype) + + if not output_type == "latent": + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0] + else: + image = latents + return StableDiffusionXLPipelineOutput(images=image) + + # apply watermark if available + if self.watermark is not None: + image = self.watermark.apply_watermark(image) + + image = self.image_processor.postprocess(image, output_type=output_type) + + if not return_dict: + final_results += [(image,)] + else: + final_results += [StableDiffusionXLPipelineOutput(images=image)] + + # Offload last model to CPU + if hasattr(self, "final_offload_hook") and self.final_offload_hook is not None: + self.final_offload_hook.offload() + + return final_results + """ + return StableDiffusionXLPipelineOutput(images=results_list) + + # Overrride to properly handle the loading and unloading of the additional text encoder. + def load_lora_weights(self, pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]], **kwargs): + # We could have accessed the unet config from `lora_state_dict()` too. We pass + # it here explicitly to be able to tell that it's coming from an SDXL + # pipeline. + state_dict, network_alphas = self.lora_state_dict( + pretrained_model_name_or_path_or_dict, + unet_config=self.unet.config, + **kwargs, + ) + self.load_lora_into_unet(state_dict, network_alphas=network_alphas, unet=self.unet) + + text_encoder_state_dict = {k: v for k, v in state_dict.items() if "text_encoder." in k} + if len(text_encoder_state_dict) > 0: + self.load_lora_into_text_encoder( + text_encoder_state_dict, + network_alphas=network_alphas, + text_encoder=self.text_encoder, + prefix="text_encoder", + lora_scale=self.lora_scale, + ) + + text_encoder_2_state_dict = {k: v for k, v in state_dict.items() if "text_encoder_2." in k} + if len(text_encoder_2_state_dict) > 0: + self.load_lora_into_text_encoder( + text_encoder_2_state_dict, + network_alphas=network_alphas, + text_encoder=self.text_encoder_2, + prefix="text_encoder_2", + lora_scale=self.lora_scale, + ) + + @classmethod + def save_lora_weights( + self, + save_directory: Union[str, os.PathLike], + unet_lora_layers: Dict[str, Union[torch.nn.Module, torch.Tensor]] = None, + text_encoder_lora_layers: Dict[str, Union[torch.nn.Module, torch.Tensor]] = None, + text_encoder_2_lora_layers: Dict[str, Union[torch.nn.Module, torch.Tensor]] = None, + is_main_process: bool = True, + weight_name: str = None, + save_function: Callable = None, + safe_serialization: bool = True, + ): + state_dict = {} + + def pack_weights(layers, prefix): + layers_weights = layers.state_dict() if isinstance(layers, torch.nn.Module) else layers + layers_state_dict = {f"{prefix}.{module_name}": param for module_name, param in layers_weights.items()} + return layers_state_dict + + state_dict.update(pack_weights(unet_lora_layers, "unet")) + + if text_encoder_lora_layers and text_encoder_2_lora_layers: + state_dict.update(pack_weights(text_encoder_lora_layers, "text_encoder")) + state_dict.update(pack_weights(text_encoder_2_lora_layers, "text_encoder_2")) + + self.write_lora_layers( + state_dict=state_dict, + save_directory=save_directory, + is_main_process=is_main_process, + weight_name=weight_name, + save_function=save_function, + safe_serialization=safe_serialization, + ) + + def _remove_text_encoder_monkey_patch(self): + self._remove_text_encoder_monkey_patch_classmethod(self.text_encoder) + self._remove_text_encoder_monkey_patch_classmethod(self.text_encoder_2) diff --git a/modules/freescale/scale_attention.py b/modules/freescale/scale_attention.py new file mode 100644 index 000000000..9e83d5067 --- /dev/null +++ b/modules/freescale/scale_attention.py @@ -0,0 +1,367 @@ +from typing import Any, Dict, Optional +import random +import torch +import torch.nn.functional as F +from einops import rearrange + + +def gaussian_kernel(kernel_size=3, sigma=1.0, channels=3): + x_coord = torch.arange(kernel_size) + gaussian_1d = torch.exp(-(x_coord - (kernel_size - 1) / 2) ** 2 / (2 * sigma ** 2)) + gaussian_1d = gaussian_1d / gaussian_1d.sum() + gaussian_2d = gaussian_1d[:, None] * gaussian_1d[None, :] + kernel = gaussian_2d[None, None, :, :].repeat(channels, 1, 1, 1) + + return kernel + +def gaussian_filter(latents, kernel_size=3, sigma=1.0): + channels = latents.shape[1] + kernel = gaussian_kernel(kernel_size, sigma, channels).to(latents.device, latents.dtype) + blurred_latents = F.conv2d(latents, kernel, padding=kernel_size//2, groups=channels) + + return blurred_latents + +def get_views(height, width, h_window_size=128, w_window_size=128, scale_factor=8): + height = int(height) + width = int(width) + h_window_stride = h_window_size // 2 + w_window_stride = w_window_size // 2 + h_window_size = int(h_window_size / scale_factor) + w_window_size = int(w_window_size / scale_factor) + h_window_stride = int(h_window_stride / scale_factor) + w_window_stride = int(w_window_stride / scale_factor) + num_blocks_height = int((height - h_window_size) / h_window_stride - 1e-6) + 2 if height > h_window_size else 1 + num_blocks_width = int((width - w_window_size) / w_window_stride - 1e-6) + 2 if width > w_window_size else 1 + total_num_blocks = int(num_blocks_height * num_blocks_width) + views = [] + for i in range(total_num_blocks): + h_start = int((i // num_blocks_width) * h_window_stride) + h_end = h_start + h_window_size + w_start = int((i % num_blocks_width) * w_window_stride) + w_end = w_start + w_window_size + + if h_end > height: + h_start = int(h_start + height - h_end) + h_end = int(height) + if w_end > width: + w_start = int(w_start + width - w_end) + w_end = int(width) + if h_start < 0: + h_end = int(h_end - h_start) + h_start = 0 + if w_start < 0: + w_end = int(w_end - w_start) + w_start = 0 + + random_jitter = True + if random_jitter: + h_jitter_range = h_window_size // 8 + w_jitter_range = w_window_size // 8 + h_jitter = 0 + w_jitter = 0 + + if (w_start != 0) and (w_end != width): + w_jitter = random.randint(-w_jitter_range, w_jitter_range) + elif (w_start == 0) and (w_end != width): + w_jitter = random.randint(-w_jitter_range, 0) + elif (w_start != 0) and (w_end == width): + w_jitter = random.randint(0, w_jitter_range) + if (h_start != 0) and (h_end != height): + h_jitter = random.randint(-h_jitter_range, h_jitter_range) + elif (h_start == 0) and (h_end != height): + h_jitter = random.randint(-h_jitter_range, 0) + elif (h_start != 0) and (h_end == height): + h_jitter = random.randint(0, h_jitter_range) + h_start += (h_jitter + h_jitter_range) + h_end += (h_jitter + h_jitter_range) + w_start += (w_jitter + w_jitter_range) + w_end += (w_jitter + w_jitter_range) + + views.append((h_start, h_end, w_start, w_end)) + return views + +def scale_forward( + self, + hidden_states: torch.FloatTensor, + attention_mask: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + timestep: Optional[torch.LongTensor] = None, + cross_attention_kwargs: Dict[str, Any] = None, + class_labels: Optional[torch.LongTensor] = None, +): + # Notice that normalization is always applied before the real computation in the following blocks. + if self.current_hw: + current_scale_num_h, current_scale_num_w = max(self.current_hw[0] // 1024, 1), max(self.current_hw[1] // 1024, 1) + else: + current_scale_num_h, current_scale_num_w = 1, 1 + + # 0. Self-Attention + if self.use_ada_layer_norm: + norm_hidden_states = self.norm1(hidden_states, timestep) + elif self.use_ada_layer_norm_zero: + norm_hidden_states, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.norm1( + hidden_states, timestep, class_labels, hidden_dtype=hidden_states.dtype + ) + else: + norm_hidden_states = self.norm1(hidden_states) + + # 2. Prepare GLIGEN inputs + cross_attention_kwargs = cross_attention_kwargs.copy() if cross_attention_kwargs is not None else {} + gligen_kwargs = cross_attention_kwargs.pop("gligen", None) + + ratio_hw = current_scale_num_h / current_scale_num_w + latent_h = int((norm_hidden_states.shape[1] * ratio_hw) ** 0.5) + latent_w = int(latent_h / ratio_hw) + scale_factor = 128 * current_scale_num_h / latent_h + if ratio_hw > 1: + sub_h = 128 + sub_w = int(128 / ratio_hw) + else: + sub_h = int(128 * ratio_hw) + sub_w = 128 + + h_jitter_range = int(sub_h / scale_factor // 8) + w_jitter_range = int(sub_w / scale_factor // 8) + views = get_views(latent_h, latent_w, sub_h, sub_w, scale_factor = scale_factor) + + current_scale_num = max(current_scale_num_h, current_scale_num_w) + global_views = [[h, w] for h in range(current_scale_num_h) for w in range(current_scale_num_w)] + + four_window = True + fourg_window = False + + if four_window: + norm_hidden_states_ = rearrange(norm_hidden_states, 'bh (h w) d -> bh h w d', h = latent_h) + norm_hidden_states_ = F.pad(norm_hidden_states_, (0, 0, w_jitter_range, w_jitter_range, h_jitter_range, h_jitter_range), 'constant', 0) + value = torch.zeros_like(norm_hidden_states_) + count = torch.zeros_like(norm_hidden_states_) + for index, view in enumerate(views): + h_start, h_end, w_start, w_end = view + local_states = norm_hidden_states_[:, h_start:h_end, w_start:w_end, :] + local_states = rearrange(local_states, 'bh h w d -> bh (h w) d') + local_output = self.attn1( + local_states, + encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None, + attention_mask=attention_mask, + **cross_attention_kwargs, + ) + local_output = rearrange(local_output, 'bh (h w) d -> bh h w d', h = int(sub_h / scale_factor)) + + value[:, h_start:h_end, w_start:w_end, :] += local_output * 1 + count[:, h_start:h_end, w_start:w_end, :] += 1 + + value = value[:, h_jitter_range:-h_jitter_range, w_jitter_range:-w_jitter_range, :] + count = count[:, h_jitter_range:-h_jitter_range, w_jitter_range:-w_jitter_range, :] + attn_output = torch.where(count>0, value/count, value) + + gaussian_local = gaussian_filter(attn_output, kernel_size=(2*current_scale_num-1), sigma=1.0) + + attn_output_global = self.attn1( + norm_hidden_states, + encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None, + attention_mask=attention_mask, + **cross_attention_kwargs, + ) + attn_output_global = rearrange(attn_output_global, 'bh (h w) d -> bh h w d', h = latent_h) + + gaussian_global = gaussian_filter(attn_output_global, kernel_size=(2*current_scale_num-1), sigma=1.0) + + attn_output = gaussian_local + (attn_output_global - gaussian_global) + attn_output = rearrange(attn_output, 'bh h w d -> bh (h w) d') + + elif fourg_window: + norm_hidden_states = rearrange(norm_hidden_states, 'bh (h w) d -> bh h w d', h = latent_h) + norm_hidden_states_ = F.pad(norm_hidden_states, (0, 0, w_jitter_range, w_jitter_range, h_jitter_range, h_jitter_range), 'constant', 0) + value = torch.zeros_like(norm_hidden_states_) + count = torch.zeros_like(norm_hidden_states_) + for index, view in enumerate(views): + h_start, h_end, w_start, w_end = view + local_states = norm_hidden_states_[:, h_start:h_end, w_start:w_end, :] + local_states = rearrange(local_states, 'bh h w d -> bh (h w) d') + local_output = self.attn1( + local_states, + encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None, + attention_mask=attention_mask, + **cross_attention_kwargs, + ) + local_output = rearrange(local_output, 'bh (h w) d -> bh h w d', h = int(sub_h / scale_factor)) + + value[:, h_start:h_end, w_start:w_end, :] += local_output * 1 + count[:, h_start:h_end, w_start:w_end, :] += 1 + + value = value[:, h_jitter_range:-h_jitter_range, w_jitter_range:-w_jitter_range, :] + count = count[:, h_jitter_range:-h_jitter_range, w_jitter_range:-w_jitter_range, :] + attn_output = torch.where(count>0, value/count, value) + + gaussian_local = gaussian_filter(attn_output, kernel_size=(2*current_scale_num-1), sigma=1.0) + + value = torch.zeros_like(norm_hidden_states) + count = torch.zeros_like(norm_hidden_states) + for index, global_view in enumerate(global_views): + h, w = global_view + global_states = norm_hidden_states[:, h::current_scale_num_h, w::current_scale_num_w, :] + global_states = rearrange(global_states, 'bh h w d -> bh (h w) d') + global_output = self.attn1( + global_states, + encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None, + attention_mask=attention_mask, + **cross_attention_kwargs, + ) + global_output = rearrange(global_output, 'bh (h w) d -> bh h w d', h = int(global_output.shape[1] ** 0.5)) + + value[:, h::current_scale_num_h, w::current_scale_num_w, :] += global_output * 1 + count[:, h::current_scale_num_h, w::current_scale_num_w, :] += 1 + + attn_output_global = torch.where(count>0, value/count, value) + + gaussian_global = gaussian_filter(attn_output_global, kernel_size=(2*current_scale_num-1), sigma=1.0) + + attn_output = gaussian_local + (attn_output_global - gaussian_global) + attn_output = rearrange(attn_output, 'bh h w d -> bh (h w) d') + + else: + attn_output = self.attn1( + norm_hidden_states, + encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None, + attention_mask=attention_mask, + **cross_attention_kwargs, + ) + + if self.use_ada_layer_norm_zero: + attn_output = gate_msa.unsqueeze(1) * attn_output + hidden_states = attn_output + hidden_states + + # 2.5 GLIGEN Control + if gligen_kwargs is not None: + hidden_states = self.fuser(hidden_states, gligen_kwargs["objs"]) + # 2.5 ends + + # 3. Cross-Attention + if self.attn2 is not None: + norm_hidden_states = ( + self.norm2(hidden_states, timestep) if self.use_ada_layer_norm else self.norm2(hidden_states) + ) + attn_output = self.attn2( + norm_hidden_states, + encoder_hidden_states=encoder_hidden_states, + attention_mask=encoder_attention_mask, + **cross_attention_kwargs, + ) + hidden_states = attn_output + hidden_states + + # 4. Feed-forward + norm_hidden_states = self.norm3(hidden_states) + + if self.use_ada_layer_norm_zero: + norm_hidden_states = norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None] + + if self._chunk_size is not None: + # "feed_forward_chunk_size" can be used to save memory + if norm_hidden_states.shape[self._chunk_dim] % self._chunk_size != 0: + raise ValueError( + f"`hidden_states` dimension to be chunked: {norm_hidden_states.shape[self._chunk_dim]} has to be divisible by chunk size: {self._chunk_size}. Make sure to set an appropriate `chunk_size` when calling `unet.enable_forward_chunking`." + ) + + num_chunks = norm_hidden_states.shape[self._chunk_dim] // self._chunk_size + ff_output = torch.cat( + [ + self.ff(hid_slice) + for hid_slice in norm_hidden_states.chunk(num_chunks, dim=self._chunk_dim) + ], + dim=self._chunk_dim, + ) + else: + ff_output = self.ff(norm_hidden_states) + + if self.use_ada_layer_norm_zero: + ff_output = gate_mlp.unsqueeze(1) * ff_output + + hidden_states = ff_output + hidden_states + + return hidden_states + +def ori_forward( + self, + hidden_states: torch.FloatTensor, + attention_mask: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + timestep: Optional[torch.LongTensor] = None, + cross_attention_kwargs: Dict[str, Any] = None, + class_labels: Optional[torch.LongTensor] = None, +): + # Notice that normalization is always applied before the real computation in the following blocks. + # 0. Self-Attention + if self.use_ada_layer_norm: + norm_hidden_states = self.norm1(hidden_states, timestep) + elif self.use_ada_layer_norm_zero: + norm_hidden_states, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.norm1( + hidden_states, timestep, class_labels, hidden_dtype=hidden_states.dtype + ) + else: + norm_hidden_states = self.norm1(hidden_states) + + # 2. Prepare GLIGEN inputs + cross_attention_kwargs = cross_attention_kwargs.copy() if cross_attention_kwargs is not None else {} + gligen_kwargs = cross_attention_kwargs.pop("gligen", None) + + attn_output = self.attn1( + norm_hidden_states, + encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None, + attention_mask=attention_mask, + **cross_attention_kwargs, + ) + + if self.use_ada_layer_norm_zero: + attn_output = gate_msa.unsqueeze(1) * attn_output + hidden_states = attn_output + hidden_states + + # 2.5 GLIGEN Control + if gligen_kwargs is not None: + hidden_states = self.fuser(hidden_states, gligen_kwargs["objs"]) + # 2.5 ends + + # 3. Cross-Attention + if self.attn2 is not None: + norm_hidden_states = ( + self.norm2(hidden_states, timestep) if self.use_ada_layer_norm else self.norm2(hidden_states) + ) + attn_output = self.attn2( + norm_hidden_states, + encoder_hidden_states=encoder_hidden_states, + attention_mask=encoder_attention_mask, + **cross_attention_kwargs, + ) + hidden_states = attn_output + hidden_states + + # 4. Feed-forward + norm_hidden_states = self.norm3(hidden_states) + + if self.use_ada_layer_norm_zero: + norm_hidden_states = norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None] + + if self._chunk_size is not None: + # "feed_forward_chunk_size" can be used to save memory + if norm_hidden_states.shape[self._chunk_dim] % self._chunk_size != 0: + raise ValueError( + f"`hidden_states` dimension to be chunked: {norm_hidden_states.shape[self._chunk_dim]} has to be divisible by chunk size: {self._chunk_size}. Make sure to set an appropriate `chunk_size` when calling `unet.enable_forward_chunking`." + ) + + num_chunks = norm_hidden_states.shape[self._chunk_dim] // self._chunk_size + ff_output = torch.cat( + [ + self.ff(hid_slice) + for hid_slice in norm_hidden_states.chunk(num_chunks, dim=self._chunk_dim) + ], + dim=self._chunk_dim, + ) + else: + ff_output = self.ff(norm_hidden_states) + + if self.use_ada_layer_norm_zero: + ff_output = gate_mlp.unsqueeze(1) * ff_output + + hidden_states = ff_output + hidden_states + + return hidden_states diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 3b6f228ba..adb047511 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -365,13 +365,26 @@ def process_decode(p: processing.StableDiffusionProcessing, output): else: width = getattr(p, 'width', 0) height = getattr(p, 'height', 0) - results = processing_vae.vae_decode( - latents = output.images, - model = model, - full_quality = p.full_quality, - width = width, - height = height, - ) + if isinstance(output.images, list): + results = [] + for i in range(len(output.images)): + result_batch = processing_vae.vae_decode( + latents = output.images[i], + model = model, + full_quality = p.full_quality, + width = width, + height = height, + ) + for result in list(result_batch): + results.append(result) + else: + results = processing_vae.vae_decode( + latents = output.images, + model = model, + full_quality = p.full_quality, + width = width, + height = height, + ) elif hasattr(output, 'images'): results = output.images else: diff --git a/modules/sd_samplers_common.py b/modules/sd_samplers_common.py index cd51043c7..723f7b181 100644 --- a/modules/sd_samplers_common.py +++ b/modules/sd_samplers_common.py @@ -40,7 +40,6 @@ def single_sample_to_image(sample, approximation=None): if approximation is None: warn_once('Unknown decode type') approximation = 0 - # normal sample is [4,64,64] try: if sample.dtype == torch.bfloat16 and (approximation == 0 or approximation == 1): sample = sample.to(torch.float16) @@ -62,6 +61,9 @@ def single_sample_to_image(sample, approximation=None): sample = sample * (5 / abs(sample_min)) """ if approximation == 2: # TAESD + if sample.shape[-1] > 128 or sample.shape[-2] > 128: + scale = 128 / max(sample.shape[-1], sample.shape[-2]) + sample = torch.nn.functional.interpolate(sample.unsqueeze(0), scale_factor=[scale, scale], mode='bilinear', align_corners=False)[0] x_sample = sd_vae_taesd.decode(sample) x_sample = (1.0 + x_sample) / 2.0 # preview requires smaller range elif shared.sd_model_type == 'sc' and approximation != 3: diff --git a/modules/sd_vae_taesd.py b/modules/sd_vae_taesd.py index 4d213ad48..a1959817c 100644 --- a/modules/sd_vae_taesd.py +++ b/modules/sd_vae_taesd.py @@ -169,6 +169,9 @@ def decode(latents): if vae is None: return latents try: + size = max(latents.shape[-1], latents.shape[-2]) + if size > 256: + return latents with devices.inference_context(): latents = latents.detach().clone().to(devices.device, dtype) if len(latents.shape) == 3: diff --git a/modules/shared.py b/modules/shared.py index 7ed14f79a..88902b978 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -873,6 +873,7 @@ def get_default_modes(): options_templates.update(options_section(('control', "Control Options"), { "control_max_units": OptionInfo(4, "Maximum number of units", gr.Slider, {"minimum": 1, "maximum": 10, "step": 1}), + "control_tiles": OptionInfo("1x1, 1x2, 1x3, 1x4, 2x1, 2x1, 2x2, 2x3, 2x4, 3x1, 3x2, 3x3, 3x4, 4x1, 4x2, 4x3, 4x4", "Tiling options"), "control_move_processor": OptionInfo(False, "Processor move to CPU after use"), "control_unload_processor": OptionInfo(False, "Processor unload after use"), })) diff --git a/modules/shared_state.py b/modules/shared_state.py index 3d3cb1ae6..a3312ec33 100644 --- a/modules/shared_state.py +++ b/modules/shared_state.py @@ -141,9 +141,9 @@ def set_current_image(self): if self.job == 'VAE': # avoid generating preview while vae is running return from modules.shared import opts, cmd_opts - if cmd_opts.lowvram or self.api: + if cmd_opts.lowvram or self.api or not opts.live_previews_enable or opts.show_progress_every_n_steps <= 0: return - if abs(self.sampling_step - self.current_image_sampling_step) >= opts.show_progress_every_n_steps and opts.live_previews_enable and opts.show_progress_every_n_steps > 0: + if abs(self.sampling_step - self.current_image_sampling_step) >= opts.show_progress_every_n_steps: self.do_set_current_image() def do_set_current_image(self): diff --git a/modules/ui_control.py b/modules/ui_control.py index 5a146a8fc..7baf74d75 100644 --- a/modules/ui_control.py +++ b/modules/ui_control.py @@ -254,7 +254,7 @@ def create_ui(_blocks: gr.Blocks=None): control_start = gr.Slider(label="CN Start", minimum=0.0, maximum=1.0, step=0.05, value=0, elem_id=f'control_unit-{i}-start') control_end = gr.Slider(label="CN End", minimum=0.0, maximum=1.0, step=0.05, value=1.0, elem_id=f'control_unit-{i}-end') control_mode = gr.Dropdown(label="CN Mode", choices=['default'], value='default', visible=False, elem_id=f'control_unit-{i}-mode') - control_tile = gr.Dropdown(label="CN Tiles", choices=['1x1', '1x2', '1x3', '1x4', '2x1', '2x1', '2x2', '2x3', '2x4', '3x1', '3x2', '3x3', '3x4', '4x1', '4x2', '4x3', '4x4'], value='1x1', visible=False, elem_id=f'control_unit-{i}-tile') + control_tile = gr.Dropdown(label="CN Tiles", choices=[x.strip() for x in shared.opts.control_tiles.split(',') if 'x' in x], value='1x1', visible=False, elem_id=f'control_unit-{i}-tile') reset_btn = ui_components.ToolButton(value=ui_symbols.reset) image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool']) image_reuse= ui_components.ToolButton(value=ui_symbols.reuse) diff --git a/scripts/freescale.py b/scripts/freescale.py new file mode 100644 index 000000000..672ceea41 --- /dev/null +++ b/scripts/freescale.py @@ -0,0 +1,130 @@ +import gradio as gr +from modules import scripts, processing, shared, sd_models + + +registered = False + + +class Script(scripts.Script): + def __init__(self): + super().__init__() + self.orig_pipe = None + self.orig_slice = None + self.orig_tile = None + self.is_img2img = False + + def title(self): + return 'FreeScale: Tuning-Free Scale Fusion' + + def show(self, is_img2img): + self.is_img2img = is_img2img + return shared.native + + def ui(self, _is_img2img): # ui elements + with gr.Row(): + gr.HTML('  FreeScale: Tuning-Free Scale Fusion
') + with gr.Row(): + cosine_scale = gr.Slider(minimum=0.1, maximum=5.0, value=2.0, label='Cosine scale') + override_sampler = gr.Checkbox(value=True, label='Override sampler') + with gr.Row(visible=self.is_img2img): + cosine_scale_bg = gr.Slider(minimum=0.1, maximum=5.0, value=1.0, label='Cosine Background') + dilate_tau = gr.Slider(minimum=1, maximum=100, value=35, label='Dilate tau') + with gr.Row(): + s1_enable = gr.Checkbox(value=True, label='1st Stage', interactive=False) + s1_scale = gr.Slider(minimum=1, maximum=8.0, value=1.0, label='Scale') + s1_restart = gr.Slider(minimum=0, maximum=1.0, value=0.75, label='Restart step') + with gr.Row(): + s2_enable = gr.Checkbox(value=True, label='2nd Stage') + s2_scale = gr.Slider(minimum=1, maximum=8.0, value=2.0, label='Scale') + s2_restart = gr.Slider(minimum=0, maximum=1.0, value=0.75, label='Restart step') + with gr.Row(): + s3_enable = gr.Checkbox(value=False, label='3rd Stage') + s3_scale = gr.Slider(minimum=1, maximum=8.0, value=3.0, label='Scale') + s3_restart = gr.Slider(minimum=0, maximum=1.0, value=0.75, label='Restart step') + with gr.Row(): + s4_enable = gr.Checkbox(value=False, label='4th Stage') + s4_scale = gr.Slider(minimum=1, maximum=8.0, value=4.0, label='Scale') + s4_restart = gr.Slider(minimum=0, maximum=1.0, value=0.75, label='Restart step') + return [cosine_scale, override_sampler, cosine_scale_bg, dilate_tau, s1_enable, s1_scale, s1_restart, s2_enable, s2_scale, s2_restart, s3_enable, s3_scale, s3_restart, s4_enable, s4_scale, s4_restart] + + def run(self, p: processing.StableDiffusionProcessing, cosine_scale, override_sampler, cosine_scale_bg, dilate_tau, s1_enable, s1_scale, s1_restart, s2_enable, s2_scale, s2_restart, s3_enable, s3_scale, s3_restart, s4_enable, s4_scale, s4_restart): # pylint: disable=arguments-differ + supported_model_list = ['sdxl'] + if shared.sd_model_type not in supported_model_list: + shared.log.warning(f'FreeScale: class={shared.sd_model.__class__.__name__} model={shared.sd_model_type} required={supported_model_list}') + return None + + if self.is_img2img: + if p.init_images is None or len(p.init_images) == 0: + shared.log.warning('FreeScale: missing input image') + return None + + from modules.freescale import StableDiffusionXLFreeScale, StableDiffusionXLFreeScaleImg2Img + self.orig_pipe = shared.sd_model + self.orig_slice = shared.opts.diffusers_vae_slicing + self.orig_tile = shared.opts.diffusers_vae_tiling + + def scale(x): + if (p.width == 0 or p.height == 0) and p.init_images is not None: + p.width, p.height = p.init_images[0].width, p.init_images[0].height + resolution = [int(8 * p.width * x // 8), int(8 * p.height * x // 8)] + return resolution + + scales = [] + resolutions_list = [] + restart_steps = [] + if s1_enable: + scales.append(s1_scale) + resolutions_list.append(scale(s1_scale)) + restart_steps.append(int(p.steps * s1_restart)) + if s2_enable and s2_scale > s1_scale: + scales.append(s2_scale) + resolutions_list.append(scale(s2_scale)) + restart_steps.append(int(p.steps * s2_restart)) + if s3_enable and s3_scale > s2_scale: + scales.append(s3_scale) + resolutions_list.append(scale(s3_scale)) + restart_steps.append(int(p.steps * s3_restart)) + if s4_enable and s4_scale > s3_scale: + scales.append(s4_scale) + resolutions_list.append(scale(s4_scale)) + restart_steps.append(int(p.steps * s4_restart)) + + p.task_args['resolutions_list'] = resolutions_list + p.task_args['cosine_scale'] = cosine_scale + p.task_args['restart_steps'] = [min(max(1, step), p.steps-1) for step in restart_steps] + if self.is_img2img: + p.task_args['cosine_scale_bg'] = cosine_scale_bg + p.task_args['dilate_tau'] = dilate_tau + p.task_args['img_path'] = p.init_images[0] + p.init_images = None + if override_sampler: + p.sampler_name = 'Euler a' + + if p.width < 1024 or p.height < 1024: + shared.log.error(f'FreeScale: width={p.width} height={p.height} minimum=1024') + return None + + if not self.is_img2img: + shared.sd_model = sd_models.switch_pipe(StableDiffusionXLFreeScale, shared.sd_model) + else: + shared.sd_model = sd_models.switch_pipe(StableDiffusionXLFreeScaleImg2Img, shared.sd_model) + shared.sd_model.enable_vae_slicing() + shared.sd_model.enable_vae_tiling() + + shared.log.info(f'FreeScale: mode={"txt" if not self.is_img2img else "img"} cosine={cosine_scale} bg={cosine_scale_bg} tau={dilate_tau} scales={scales} resolutions={resolutions_list} steps={restart_steps} sampler={p.sampler_name}') + resolutions = ','.join([f'{x[0]}x{x[1]}' for x in resolutions_list]) + steps = ','.join([str(x) for x in restart_steps]) + p.extra_generation_params["FreeScale"] = f'cosine {cosine_scale} resolutions {resolutions} steps {steps}' + + def after(self, p: processing.StableDiffusionProcessing, processed: processing.Processed, *args): # pylint: disable=arguments-differ, unused-argument + if self.orig_pipe is None: + return processed + # restore pipeline + if shared.sd_model_type == "sdxl": + shared.sd_model = self.orig_pipe + self.orig_pipe = None + if not self.orig_slice: + shared.sd_model.disable_vae_slicing() + if not self.orig_tile: + shared.sd_model.disable_vae_tiling() + return processed From 005192549876b90de83cbadf838aa827aa7f7ba6 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 15 Dec 2024 13:12:35 -0500 Subject: [PATCH 093/249] lint updates Signed-off-by: Vladimir Mandic --- .pylintrc | 1 + .ruff.toml | 1 + modules/control/run.py | 332 +++++++++++++++++++++------------------- modules/lora/network.py | 1 - modules/shared.py | 2 +- 5 files changed, 175 insertions(+), 162 deletions(-) diff --git a/.pylintrc b/.pylintrc index 59f1cb127..ad42ddd13 100644 --- a/.pylintrc +++ b/.pylintrc @@ -13,6 +13,7 @@ ignore-paths=/usr/lib/.*$, modules/control/units, modules/ctrlx, modules/dml, + modules/freescale, modules/ggml, modules/hidiffusion, modules/hijack, diff --git a/.ruff.toml b/.ruff.toml index c2d4a6f9a..4bab64260 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -7,6 +7,7 @@ exclude = [ "modules/consistory", "modules/control/proc", "modules/control/units", + "modules/freescale", "modules/ggml", "modules/hidiffusion", "modules/hijack", diff --git a/modules/control/run.py b/modules/control/run.py index ac1ff233d..e780b9bae 100644 --- a/modules/control/run.py +++ b/modules/control/run.py @@ -45,6 +45,167 @@ def terminate(msg): return msg +def set_pipe(p, has_models, unit_type, selected_models, active_model, active_strength, control_conditioning, control_guidance_start, control_guidance_end, inits): + global pipe, instance # pylint: disable=global-statement + pipe = None + if has_models: + p.ops.append('control') + p.extra_generation_params["Control type"] = unit_type # overriden later with pretty-print + p.extra_generation_params["Control model"] = ';'.join([(m.model_id or '') for m in active_model if m.model is not None]) + p.extra_generation_params["Control conditioning"] = control_conditioning if isinstance(control_conditioning, list) else [control_conditioning] + p.extra_generation_params['Control start'] = control_guidance_start if isinstance(control_guidance_start, list) else [control_guidance_start] + p.extra_generation_params['Control end'] = control_guidance_end if isinstance(control_guidance_end, list) else [control_guidance_end] + p.extra_generation_params["Control conditioning"] = ';'.join([str(c) for c in p.extra_generation_params["Control conditioning"]]) + p.extra_generation_params['Control start'] = ';'.join([str(c) for c in p.extra_generation_params['Control start']]) + p.extra_generation_params['Control end'] = ';'.join([str(c) for c in p.extra_generation_params['Control end']]) + if unit_type == 't2i adapter' and has_models: + p.extra_generation_params["Control type"] = 'T2I-Adapter' + p.task_args['adapter_conditioning_scale'] = control_conditioning + instance = t2iadapter.AdapterPipeline(selected_models, shared.sd_model) + pipe = instance.pipeline + if inits is not None: + shared.log.warning('Control: T2I-Adapter does not support separate init image') + elif unit_type == 'controlnet' and has_models: + p.extra_generation_params["Control type"] = 'ControlNet' + p.task_args['controlnet_conditioning_scale'] = control_conditioning + p.task_args['control_guidance_start'] = control_guidance_start + p.task_args['control_guidance_end'] = control_guidance_end + p.task_args['guess_mode'] = p.guess_mode + instance = controlnet.ControlNetPipeline(selected_models, shared.sd_model, p=p) + pipe = instance.pipeline + elif unit_type == 'xs' and has_models: + p.extra_generation_params["Control type"] = 'ControlNet-XS' + p.controlnet_conditioning_scale = control_conditioning + p.control_guidance_start = control_guidance_start + p.control_guidance_end = control_guidance_end + instance = xs.ControlNetXSPipeline(selected_models, shared.sd_model) + pipe = instance.pipeline + if inits is not None: + shared.log.warning('Control: ControlNet-XS does not support separate init image') + elif unit_type == 'lite' and has_models: + p.extra_generation_params["Control type"] = 'ControlLLLite' + p.controlnet_conditioning_scale = control_conditioning + instance = lite.ControlLLitePipeline(shared.sd_model) + pipe = instance.pipeline + if inits is not None: + shared.log.warning('Control: ControlLLLite does not support separate init image') + elif unit_type == 'reference' and has_models: + p.extra_generation_params["Control type"] = 'Reference' + p.extra_generation_params["Control attention"] = p.attention + p.task_args['reference_attn'] = 'Attention' in p.attention + p.task_args['reference_adain'] = 'Adain' in p.attention + p.task_args['attention_auto_machine_weight'] = p.query_weight + p.task_args['gn_auto_machine_weight'] = p.adain_weight + p.task_args['style_fidelity'] = p.fidelity + instance = reference.ReferencePipeline(shared.sd_model) + pipe = instance.pipeline + if inits is not None: + shared.log.warning('Control: ControlNet-XS does not support separate init image') + else: # run in txt2img/img2img mode + if len(active_strength) > 0: + p.strength = active_strength[0] + pipe = shared.sd_model + instance = None + debug(f'Control: run type={unit_type} models={has_models} pipe={pipe.__class__.__name__ if pipe is not None else None}') + return pipe + + +def check_active(p, unit_type, units): + active_process: List[processors.Processor] = [] # all active preprocessors + active_model: List[Union[controlnet.ControlNet, xs.ControlNetXS, t2iadapter.Adapter]] = [] # all active models + active_strength: List[float] = [] # strength factors for all active models + active_start: List[float] = [] # start step for all active models + active_end: List[float] = [] # end step for all active models + num_units = 0 + for u in units: + if u.type != unit_type: + continue + num_units += 1 + debug(f'Control unit: i={num_units} type={u.type} enabled={u.enabled}') + if not u.enabled: + if u.controlnet is not None and u.controlnet.model is not None: + debug(f'Control unit offload: model="{u.controlnet.model_id}" device={devices.cpu}') + sd_models.move_model(u.controlnet.model, devices.cpu) + continue + if u.controlnet is not None and u.controlnet.model is not None: + debug(f'Control unit offload: model="{u.controlnet.model_id}" device={devices.device}') + sd_models.move_model(u.controlnet.model, devices.device) + if unit_type == 't2i adapter' and u.adapter.model is not None: + active_process.append(u.process) + active_model.append(u.adapter) + active_strength.append(float(u.strength)) + p.adapter_conditioning_factor = u.factor + shared.log.debug(f'Control T2I-Adapter unit: i={num_units} process="{u.process.processor_id}" model="{u.adapter.model_id}" strength={u.strength} factor={u.factor}') + elif unit_type == 'controlnet' and u.controlnet.model is not None: + active_process.append(u.process) + active_model.append(u.controlnet) + active_strength.append(float(u.strength)) + active_start.append(float(u.start)) + active_end.append(float(u.end)) + p.guess_mode = u.guess + if isinstance(u.mode, str): + p.control_mode = u.choices.index(u.mode) if u.mode in u.choices else 0 + p.is_tile = p.is_tile or 'tile' in u.mode.lower() + p.control_tile = u.tile + p.extra_generation_params["Control mode"] = u.mode + shared.log.debug(f'Control ControlNet unit: i={num_units} process="{u.process.processor_id}" model="{u.controlnet.model_id}" strength={u.strength} guess={u.guess} start={u.start} end={u.end} mode={u.mode}') + elif unit_type == 'xs' and u.controlnet.model is not None: + active_process.append(u.process) + active_model.append(u.controlnet) + active_strength.append(float(u.strength)) + active_start.append(float(u.start)) + active_end.append(float(u.end)) + shared.log.debug(f'Control ControlNet-XS unit: i={num_units} process={u.process.processor_id} model={u.controlnet.model_id} strength={u.strength} guess={u.guess} start={u.start} end={u.end}') + elif unit_type == 'lite' and u.controlnet.model is not None: + active_process.append(u.process) + active_model.append(u.controlnet) + active_strength.append(float(u.strength)) + shared.log.debug(f'Control ControlLLite unit: i={num_units} process={u.process.processor_id} model={u.controlnet.model_id} strength={u.strength} guess={u.guess} start={u.start} end={u.end}') + elif unit_type == 'reference': + p.override = u.override + p.attention = u.attention + p.query_weight = float(u.query_weight) + p.adain_weight = float(u.adain_weight) + p.fidelity = u.fidelity + shared.log.debug('Control Reference unit') + else: + if u.process.processor_id is not None: + active_process.append(u.process) + shared.log.debug(f'Control process unit: i={num_units} process={u.process.processor_id}') + active_strength.append(float(u.strength)) + debug(f'Control active: process={len(active_process)} model={len(active_model)}') + return active_process, active_model, active_strength, active_start, active_end + + +def check_enabled(p, unit_type, units, active_model, active_strength, active_start, active_end): + has_models = False + selected_models: List[Union[controlnet.ControlNetModel, xs.ControlNetXSModel, t2iadapter.AdapterModel]] = None + control_conditioning = None + control_guidance_start = None + control_guidance_end = None + if unit_type == 't2i adapter' or unit_type == 'controlnet' or unit_type == 'xs' or unit_type == 'lite': + if len(active_model) == 0: + selected_models = None + elif len(active_model) == 1: + selected_models = active_model[0].model if active_model[0].model is not None else None + p.is_tile = p.is_tile or 'tile' in active_model[0].model_id.lower() + has_models = selected_models is not None + control_conditioning = active_strength[0] if len(active_strength) > 0 else 1 # strength or list[strength] + control_guidance_start = active_start[0] if len(active_start) > 0 else 0 + control_guidance_end = active_end[0] if len(active_end) > 0 else 1 + else: + selected_models = [m.model for m in active_model if m.model is not None] + has_models = len(selected_models) > 0 + control_conditioning = active_strength[0] if len(active_strength) == 1 else list(active_strength) # strength or list[strength] + control_guidance_start = active_start[0] if len(active_start) == 1 else list(active_start) + control_guidance_end = active_end[0] if len(active_end) == 1 else list(active_end) + elif unit_type == 'reference': + has_models = any(u.enabled for u in units if u.type == 'reference') + else: + pass + return has_models, selected_models, control_conditioning, control_guidance_start, control_guidance_end + + def control_set(kwargs): if kwargs: global p_extra_args # pylint: disable=global-statement @@ -88,16 +249,11 @@ def control_run(state: str = '', if u.process is not None and u.process.override is None and u.override is not None: u.process.override = u.override - global instance, pipe, original_pipeline # pylint: disable=global-statement + global pipe, original_pipeline # pylint: disable=global-statement debug(f'Control: type={unit_type} input={inputs} init={inits} type={input_type}') if inputs is None or (type(inputs) is list and len(inputs) == 0): inputs = [None] output_images: List[Image.Image] = [] # output images - active_process: List[processors.Processor] = [] # all active preprocessors - active_model: List[Union[controlnet.ControlNet, xs.ControlNetXS, t2iadapter.Adapter]] = [] # all active models - active_strength: List[float] = [] # strength factors for all active models - active_start: List[float] = [] # start step for all active models - active_end: List[float] = [] # end step for all active models processed_image: Image.Image = None # last processed image if mask is not None and input_type == 0: input_type = 1 # inpaint always requires control_image @@ -226,160 +382,17 @@ def control_run(state: str = '', unit_type = unit_type.strip().lower() if unit_type is not None else '' t0 = time.time() - num_units = 0 - for u in units: - if u.type != unit_type: - continue - num_units += 1 - debug(f'Control unit: i={num_units} type={u.type} enabled={u.enabled}') - if not u.enabled: - if u.controlnet is not None and u.controlnet.model is not None: - debug(f'Control unit offload: model="{u.controlnet.model_id}" device={devices.cpu}') - sd_models.move_model(u.controlnet.model, devices.cpu) - continue - if u.controlnet is not None and u.controlnet.model is not None: - debug(f'Control unit offload: model="{u.controlnet.model_id}" device={devices.device}') - sd_models.move_model(u.controlnet.model, devices.device) - if unit_type == 't2i adapter' and u.adapter.model is not None: - active_process.append(u.process) - active_model.append(u.adapter) - active_strength.append(float(u.strength)) - p.adapter_conditioning_factor = u.factor - shared.log.debug(f'Control T2I-Adapter unit: i={num_units} process="{u.process.processor_id}" model="{u.adapter.model_id}" strength={u.strength} factor={u.factor}') - elif unit_type == 'controlnet' and u.controlnet.model is not None: - active_process.append(u.process) - active_model.append(u.controlnet) - active_strength.append(float(u.strength)) - active_start.append(float(u.start)) - active_end.append(float(u.end)) - p.guess_mode = u.guess - if isinstance(u.mode, str): - p.control_mode = u.choices.index(u.mode) if u.mode in u.choices else 0 - p.is_tile = p.is_tile or 'tile' in u.mode.lower() - p.control_tile = u.tile - p.extra_generation_params["Control mode"] = u.mode - shared.log.debug(f'Control ControlNet unit: i={num_units} process="{u.process.processor_id}" model="{u.controlnet.model_id}" strength={u.strength} guess={u.guess} start={u.start} end={u.end} mode={u.mode}') - elif unit_type == 'xs' and u.controlnet.model is not None: - active_process.append(u.process) - active_model.append(u.controlnet) - active_strength.append(float(u.strength)) - active_start.append(float(u.start)) - active_end.append(float(u.end)) - shared.log.debug(f'Control ControlNet-XS unit: i={num_units} process={u.process.processor_id} model={u.controlnet.model_id} strength={u.strength} guess={u.guess} start={u.start} end={u.end}') - elif unit_type == 'lite' and u.controlnet.model is not None: - active_process.append(u.process) - active_model.append(u.controlnet) - active_strength.append(float(u.strength)) - shared.log.debug(f'Control ControlLLite unit: i={num_units} process={u.process.processor_id} model={u.controlnet.model_id} strength={u.strength} guess={u.guess} start={u.start} end={u.end}') - elif unit_type == 'reference': - p.override = u.override - p.attention = u.attention - p.query_weight = float(u.query_weight) - p.adain_weight = float(u.adain_weight) - p.fidelity = u.fidelity - shared.log.debug('Control Reference unit') - else: - if u.process.processor_id is not None: - active_process.append(u.process) - shared.log.debug(f'Control process unit: i={num_units} process={u.process.processor_id}') - active_strength.append(float(u.strength)) - debug(f'Control active: process={len(active_process)} model={len(active_model)}') + + active_process, active_model, active_strength, active_start, active_end = check_active(p, unit_type, units) + has_models, selected_models, control_conditioning, control_guidance_start, control_guidance_end = check_enabled(p, unit_type, units, active_model, active_strength, active_start, active_end) processed: processing.Processed = None image_txt = '' info_txt = [] - has_models = False - selected_models: List[Union[controlnet.ControlNetModel, xs.ControlNetXSModel, t2iadapter.AdapterModel]] = None - control_conditioning = None - control_guidance_start = None - control_guidance_end = None - if unit_type == 't2i adapter' or unit_type == 'controlnet' or unit_type == 'xs' or unit_type == 'lite': - if len(active_model) == 0: - selected_models = None - elif len(active_model) == 1: - selected_models = active_model[0].model if active_model[0].model is not None else None - p.is_tile = p.is_tile or 'tile' in active_model[0].model_id.lower() - has_models = selected_models is not None - control_conditioning = active_strength[0] if len(active_strength) > 0 else 1 # strength or list[strength] - control_guidance_start = active_start[0] if len(active_start) > 0 else 0 - control_guidance_end = active_end[0] if len(active_end) > 0 else 1 - else: - selected_models = [m.model for m in active_model if m.model is not None] - has_models = len(selected_models) > 0 - control_conditioning = active_strength[0] if len(active_strength) == 1 else list(active_strength) # strength or list[strength] - control_guidance_start = active_start[0] if len(active_start) == 1 else list(active_start) - control_guidance_end = active_end[0] if len(active_end) == 1 else list(active_end) - elif unit_type == 'reference': - has_models = any(u.enabled for u in units if u.type == 'reference') - else: - pass + p.is_tile = p.is_tile and has_models - def set_pipe(): - global pipe, instance # pylint: disable=global-statement - pipe = None - if has_models: - p.ops.append('control') - p.extra_generation_params["Control type"] = unit_type # overriden later with pretty-print - p.extra_generation_params["Control model"] = ';'.join([(m.model_id or '') for m in active_model if m.model is not None]) - p.extra_generation_params["Control conditioning"] = control_conditioning if isinstance(control_conditioning, list) else [control_conditioning] - p.extra_generation_params['Control start'] = control_guidance_start if isinstance(control_guidance_start, list) else [control_guidance_start] - p.extra_generation_params['Control end'] = control_guidance_end if isinstance(control_guidance_end, list) else [control_guidance_end] - p.extra_generation_params["Control conditioning"] = ';'.join([str(c) for c in p.extra_generation_params["Control conditioning"]]) - p.extra_generation_params['Control start'] = ';'.join([str(c) for c in p.extra_generation_params['Control start']]) - p.extra_generation_params['Control end'] = ';'.join([str(c) for c in p.extra_generation_params['Control end']]) - if unit_type == 't2i adapter' and has_models: - p.extra_generation_params["Control type"] = 'T2I-Adapter' - p.task_args['adapter_conditioning_scale'] = control_conditioning - instance = t2iadapter.AdapterPipeline(selected_models, shared.sd_model) - pipe = instance.pipeline - if inits is not None: - shared.log.warning('Control: T2I-Adapter does not support separate init image') - elif unit_type == 'controlnet' and has_models: - p.extra_generation_params["Control type"] = 'ControlNet' - p.task_args['controlnet_conditioning_scale'] = control_conditioning - p.task_args['control_guidance_start'] = control_guidance_start - p.task_args['control_guidance_end'] = control_guidance_end - p.task_args['guess_mode'] = p.guess_mode - instance = controlnet.ControlNetPipeline(selected_models, shared.sd_model, p=p) - pipe = instance.pipeline - elif unit_type == 'xs' and has_models: - p.extra_generation_params["Control type"] = 'ControlNet-XS' - p.controlnet_conditioning_scale = control_conditioning - p.control_guidance_start = control_guidance_start - p.control_guidance_end = control_guidance_end - instance = xs.ControlNetXSPipeline(selected_models, shared.sd_model) - pipe = instance.pipeline - if inits is not None: - shared.log.warning('Control: ControlNet-XS does not support separate init image') - elif unit_type == 'lite' and has_models: - p.extra_generation_params["Control type"] = 'ControlLLLite' - p.controlnet_conditioning_scale = control_conditioning - instance = lite.ControlLLitePipeline(shared.sd_model) - pipe = instance.pipeline - if inits is not None: - shared.log.warning('Control: ControlLLLite does not support separate init image') - elif unit_type == 'reference' and has_models: - p.extra_generation_params["Control type"] = 'Reference' - p.extra_generation_params["Control attention"] = p.attention - p.task_args['reference_attn'] = 'Attention' in p.attention - p.task_args['reference_adain'] = 'Adain' in p.attention - p.task_args['attention_auto_machine_weight'] = p.query_weight - p.task_args['gn_auto_machine_weight'] = p.adain_weight - p.task_args['style_fidelity'] = p.fidelity - instance = reference.ReferencePipeline(shared.sd_model) - pipe = instance.pipeline - if inits is not None: - shared.log.warning('Control: ControlNet-XS does not support separate init image') - else: # run in txt2img/img2img mode - if len(active_strength) > 0: - p.strength = active_strength[0] - pipe = shared.sd_model - instance = None - debug(f'Control: run type={unit_type} models={has_models} pipe={pipe.__class__.__name__ if pipe is not None else None}') - return pipe - - pipe = set_pipe() + pipe = set_pipe(p, has_models, unit_type, selected_models, active_model, active_strength, control_conditioning, control_guidance_start, control_guidance_end, inits) debug(f'Control pipeline: class={pipe.__class__.__name__} args={vars(p)}') t1, t2, t3 = time.time(), 0, 0 status = True @@ -433,7 +446,7 @@ def set_pipe(): while status: if pipe is None: # pipe may have been reset externally - pipe = set_pipe() + pipe = set_pipe(p, has_models, unit_type, selected_models, active_model, active_strength, control_conditioning, control_guidance_start, control_guidance_end, inits) debug(f'Control pipeline reinit: class={pipe.__class__.__name__}') processed_image = None if frame is not None: @@ -578,7 +591,7 @@ def set_pipe(): elif 'image' in possible: p.task_args['image'] = [p.init_images] if isinstance(p.init_images, Image.Image) else p.init_images if 'control_mode' in possible: - p.task_args['control_mode'] = p.control_mode + p.task_args['control_mode'] = getattr(p, 'control_mode', None) if 'strength' in possible: p.task_args['strength'] = p.denoising_strength p.init_images = None @@ -638,8 +651,8 @@ def set_pipe(): if unit_type == 'lite': p.init_image = [input_image] instance.apply(selected_models, processed_image, control_conditioning) - if p.control_mode is not None: - p.task_args['control_mode'] = p.control_mode + if getattr(p, 'control_mode', None) is not None: + p.task_args['control_mode'] = getattr(p, 'control_mode', None) if hasattr(p, 'init_images') and p.init_images is None: # delete empty del p.init_images @@ -770,5 +783,4 @@ def set_pipe(): html_txt = html_txt + infotext_to_html(info_txt[0]) if is_generator: yield (output_images, blended_image, html_txt, output_filename) - else: - return (output_images, blended_image, html_txt, output_filename) + return (output_images, blended_image, html_txt, output_filename) diff --git a/modules/lora/network.py b/modules/lora/network.py index 8e6f87368..97feb76f1 100644 --- a/modules/lora/network.py +++ b/modules/lora/network.py @@ -2,7 +2,6 @@ import enum from typing import Union from collections import namedtuple - from modules import sd_models, hashes, shared diff --git a/modules/shared.py b/modules/shared.py index 88902b978..23865ab63 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -490,7 +490,7 @@ def get_default_modes(): "advanced_sep": OptionInfo("

Advanced Options

", "", gr.HTML), "sd_checkpoint_autoload": OptionInfo(True, "Model autoload on start"), "sd_checkpoint_autodownload": OptionInfo(True, "Model auto-download on demand"), - "stream_load": OptionInfo(False, "Load models using stream loading method", gr.Checkbox, {"visible": not native }), + "stream_load": OptionInfo(False, "Model load using streams", gr.Checkbox), "diffusers_eval": OptionInfo(True, "Force model eval", gr.Checkbox, {"visible": False }), "diffusers_to_gpu": OptionInfo(False, "Load model directly to GPU"), "disable_accelerate": OptionInfo(False, "Disable accelerate", gr.Checkbox, {"visible": False }), From 42a28cb05a51f2e184eb832fb46295ace2a185c2 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 15 Dec 2024 13:28:40 -0500 Subject: [PATCH 094/249] update requirements Signed-off-by: Vladimir Mandic --- TODO.md | 2 +- requirements.txt | 8 ++++---- wiki | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/TODO.md b/TODO.md index 63088d39f..996da5ad9 100644 --- a/TODO.md +++ b/TODO.md @@ -17,12 +17,12 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma - SANA: - LTX-Video: - TorchAO: -- ControlNetUnion/ControlNetPromax: ## Other - IPAdapter negative: - Control API enhance scripts compatibility +- PixelSmith: ## Workaround in place diff --git a/requirements.txt b/requirements.txt index da8870468..100c57dfb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -41,18 +41,18 @@ torchsde==0.2.6 antlr4-python3-runtime==4.9.3 requests==2.32.3 tqdm==4.66.5 -accelerate==1.1.1 +accelerate==1.2.1 opencv-contrib-python-headless==4.9.0.80 einops==0.4.1 gradio==3.43.2 -huggingface_hub==0.26.2 +huggingface_hub==0.26.5 numexpr==2.8.8 numpy==1.26.4 numba==0.59.1 protobuf==4.25.3 pytorch_lightning==1.9.4 -tokenizers==0.20.3 -transformers==4.46.3 +tokenizers==0.21.0 +transformers==4.47.0 urllib3==1.26.19 Pillow==10.4.0 pillow-jxl-plugin==1.3.0 diff --git a/wiki b/wiki index 8d63a0f04..a4eaad83c 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 8d63a0f04687f24c4ef413f231970087f167175c +Subproject commit a4eaad83ccb8e82cb91fde4c038877616ed012d6 From 1215d86a7b56dd8d5c143cb5c45c2c9b05eb1c1a Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 16 Dec 2024 11:30:15 -0500 Subject: [PATCH 095/249] add sana Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 13 ++- html/reference.json | 13 +++ installer.py | 2 +- ...rge-Model--Sana_1600M_1024px_diffusers.jpg | Bin 0 -> 53061 bytes modules/model_flux.py | 6 +- modules/model_omnigen.py | 3 +- modules/model_sana.py | 25 ++++ modules/model_te.py | 3 +- modules/modeldata.py | 83 +++++++------ modules/pag/__init__.py | 7 +- modules/processing_vae.py | 3 +- modules/schedulers/scheduler_dpm_flowmatch.py | 3 +- modules/sd_detect.py | 2 + modules/sd_models.py | 3 + modules/sd_samplers.py | 9 +- modules/sd_samplers_common.py | 1 + modules/sd_samplers_diffusers.py | 109 +++++++++--------- modules/shared_items.py | 21 ++-- modules/ui_sections.py | 4 +- 19 files changed, 188 insertions(+), 122 deletions(-) create mode 100644 models/Reference/Efficient-Large-Model--Sana_1600M_1024px_diffusers.jpg create mode 100644 modules/model_sana.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 35d3fcbe9..3a07f154c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,19 @@ # Change Log for SD.Next -## Update for 2024-12-15 +## Update for 2024-12-16 + +- Sana: both 1.6B and 0.6B +- ControlNet: better Union results, support for ProMax and Tile +- FreeScale: run optimized iterative generation of images at different scales +- Samplers: UniPC, DEIS, SA, DPM-Multistep: add FlowMatch sigma method and prediction type ### New models and integrations +- [NVLabs Sana](https://huggingface.co/Efficient-Large-Model/Sana_1600M_1024px) + **Sana** can synthesize high-resolution images with strong text-image alignment by using **Gemma2** as text-encoder + support for both 1.6B and 0.6B models + to use, select from *networks -> models -> reference* and models will be auto-downloaded on first use + *reference values*: sampler: default, width/height: 1024, guidance scale: 4.5, attention guidance: 3.0, adaptive scaling: 0.0 - [Flux Tools](https://blackforestlabs.ai/flux-1-tools/) **Redux** is actually a tool, **Fill** is inpaint/outpaint optimized version of *Flux-dev* **Canny** & **Depth** are optimized versions of *Flux-dev* for their respective tasks: they are *not* ControlNets that work on top of a model @@ -98,6 +108,7 @@ - **IPEX**: update to IPEX 2.5.10+xpu - **OpenVINO**: update to 2024.5.0 - **Sampler** improvements + - UniPC, DEIS, SA, DPM-Multistep: allow FlowMatch method - Euler FlowMatch: add sigma methods (*karras/exponential/betas*) - Euler FlowMatch: allow using timestep presets to set sigmas - DPM FlowMatch: update all and add sigma methods diff --git a/html/reference.json b/html/reference.json index 4a549586f..8a0965697 100644 --- a/html/reference.json +++ b/html/reference.json @@ -180,6 +180,19 @@ "extras": "sampler: Default, cfg_scale: 3.5" }, + "NVLabs Sana 1.6B": { + "path": "Efficient-Large-Model/Sana_1600M_1024px_diffusers", + "desc": "Sana is a text-to-image framework that can efficiently generate images up to 4096 × 4096 resolution. Sana can synthesize high-resolution, high-quality images with strong text-image alignment at a remarkably fast speed, deployable on laptop GPU.", + "preview": "Efficient-Large-Model--Sana_1600M_1024px_diffusers.jpg", + "skip": true + }, + "NVLabs Sana 0.6B": { + "path": "Efficient-Large-Model/Sana_600M_1024px_diffusers", + "desc": "Sana is a text-to-image framework that can efficiently generate images up to 4096 × 4096 resolution. Sana can synthesize high-resolution, high-quality images with strong text-image alignment at a remarkably fast speed, deployable on laptop GPU.", + "preview": "Efficient-Large-Model--Sana_1600M_1024px_diffusers.jpg", + "skip": true + }, + "VectorSpaceLab OmniGen v1": { "path": "Shitao/OmniGen-v1", "desc": "OmniGen is a unified image generation model that can generate a wide range of images from multi-modal prompts. It is designed to be simple, flexible and easy to use.", diff --git a/installer.py b/installer.py index 18a8ad1f1..a12b09d4d 100644 --- a/installer.py +++ b/installer.py @@ -459,7 +459,7 @@ def check_python(supported_minors=[9, 10, 11, 12], reason=None): def check_diffusers(): if args.skip_all or args.skip_requirements: return - sha = '63243406ba5510c10d5cac931882918ceba926f9' # diffusers commit hash + sha = '5fb3a985173efaae7ff381b9040c386751d643da' # diffusers commit hash pkg = pkg_resources.working_set.by_key.get('diffusers', None) minor = int(pkg.version.split('.')[1] if pkg is not None else 0) cur = opts.get('diffusers_version', '') if minor > 0 else '' diff --git a/models/Reference/Efficient-Large-Model--Sana_1600M_1024px_diffusers.jpg b/models/Reference/Efficient-Large-Model--Sana_1600M_1024px_diffusers.jpg new file mode 100644 index 0000000000000000000000000000000000000000..654f854034544bc8fafa5a0de21b54c4b3862709 GIT binary patch literal 53061 zcmbTdby!=`(=Hkc6p9v#yK8Zm5L$}6JH;gwcZaq>3&EY@9w1QMp~aux30013;f$!EcLS(j?jH4JUj8#R|EnRrLPkM-jfVaP1M}sC zy0?H=NXW>qP>@kkQBYpa4tN;{px~k6)AD?H{Z8Esjn0LDHzY0>o&IChUqX%X69ztW z*RO9dh=@r@$rzcKSypK3^*v_X=~_Jl(7m0S zV0MSk0OTGC9d@&2%_wfXx#sH=tH1Z@$r;O6^Una7YQjGi;lOP&^Q(%&2g1^K_f6&5 zkFz}>>-#j}aX-0D6y}NM#1oT7+`ClrU0!&z>k*?$%sEA0+@Z1hiDNBjvtMXJ`WZlG zG9rD?2KpDR)b%v%fxXIMY4gN5L7mU~RNIyjam62$7O>55O15KtQeO+&A_c7=<;Tmo z=HosWIT4LVpL%b(BU7i6(!xS)zqv&E`GFm`$Lo{&8#Tof--X518Ny zqQ1jVv6D?&m%Oz8F5R&L;joJJ3*1A19N(d)#N8-hUSq zN@Q`@+CoUkc2d=7Qh$b9N>+&XXkr}U$2MCZ=bMjPQanhllVV;pSZ&c~!m??8SKC6^ z+TmMM6;QHic4_Cyc3AfQ+gvtFJB+i(I0=+8%#OKtb|9%!?&BVQz&Hb|oyumpSEJTV zv0&CTlFUAS21sezzo}p1dj=?ja?7`?LVZ>6l=^x&%w5A+T_j(#O6HiGXS`S^&EOnc zHP$~`^=g}hb*mj%U=!w6!8vzu4?1rP_bUNVw~VXoMA4>0QyQ6jrS)Vxj@{T0K|@OQ z;H#%ILW+0M!_NRj;Slo}*jW~IvUGJL&zMmo_vNc&1YHx>?KWDSZ$)OVo}J)6Y6gu0P(AWKX6tSwF$6AMlvBv!G z8#RIqHlwte2N6Zp-xSkVIV+uPTz}*1Q6HVxC)`sZ`%YsNcmmvqiAxKQ4n7|>c`P&@ zrLYi*ygq6Kf(aV3Sku;7x+}PK?+_6h$q6!^ih94Zl_Csn^y$90xHZy@1b{cb7cee_ zVl-J~q1v;lZ_Ff#uo_dC6t51E+r@3b<40s~)%q6(4u@R>fv0r)suY3;%hr)1vg+|m zP1_OMQNs0Jz1j{wViemyQOHy(b{+V=I5Vm~H^Lt6vyx8a0Jy5){ZRfSh>d!^6fNf= zy{lflidZ@YB$gy;zrSzSFh4VCV;i_@*-!3AC2c^PblSh6d8)FJHGe|>Bt5M8)FJy6 zT-$;44A?N90_|S}?O^KEGiDXa>9H_I1NM}8F!hAm`Nj|O^r_?-&Kci!R~0FZr~uTn zd~&e&!1bHhwH2~ba5ZEopCqOdZ;3GT>tJf5$I&SFUzAq-nUbnjZZu3>65LR(>U!s( z5f@2HRLxd+UUjXIGC}pH)~cuvY(Tk@AT~ggk@rpZDxxuKk!*D{M3EQebDHFcjA1(9 zfPECS>g6qd-Tn;t6ZF&u@TK(Ge&9Stw-EV%^{m`1ubjF3I(|TV2JDZa?Dm>8K6tzIO0PDb4*wV6IEYUzSZC1VVH&rC2u2*^CDgBD>GdoYn0f{ z4M&&Z%iD2#xFku`2A}M3NE-bnxvr?}^3|@KEke{WF4{YEyJST#BdWLI_iQui?$mTI z>L;dpa#UPGhL(9Bu}=4yBa~o4IBplvShWes#bq{LvD!dm4HdD zyjEJ&#QU3W>xdT=Im}eHbVPkvR(c4CEC5jk{G zc8K1`E#)@l23Qc{MzG#dC_Mu>?Y(*ouyoY=pmq~J%>jLEvSM!zFA7QP2g5is9f%&& zhDbNiHskunf0UdKe_`L#i?B}?KHqv^a5^m`QorW^WxJFq;V_`KXu<9w47!}^=xqjJ z?4IoT=6w)n-{>sT$Gr|W4JZYcwN=>sEK|iLA7mDv7TPI{zj8)<w5oYT)>wWslfY~lfFGl>JEcHvt z?Mf?YPp?2+aSes~j368I;sY?2rXMJYGEKR@IAJf))(>P@;jYf5h!L`mm2v7FZ$Y?j zlmsrG0jUWMQ?fTYN~eeBY>8=_#e`S73tx&Mm+Q{}uHZ*C9egaZf?15WPg?HdHyuXw zPZD$>y)wZ;qO=KNcufncHswIU7NYoLuEKP@Ozvr5Bih}N#pSnWK&1g$@mH>kMV`un zWFd4uIu;T2%VT-%04)0VjPkBMWoZuU0al`g2?3@rN`Rp3T&>dmH<(Gn8Mbn|mHVLM z$?@Vos*J{wCn9U8t#DpuQ>4kMtzbsz0r|yE8tpNv@=3FC2jHZ7cXuq0xnDg4KnlGP z__~Fho=z0+sa5~YO0d;|g1U6jo=~3wGYBcexCf?CLCwythP7wFzex#(2^IfQjM+f! z7n@z-={=Z`-frzSl4vn`YPOE8tJpD_6VWFf(W#y7CQnMuJ+}~O8K@mohP%jo{|p#| z!+oiAcnsL#1I24q;-`(p=CSCkoxuRsKN?Ls1Q_aa<{>KT8oe>*H(XGIx~Q#Rn|+iM zf{%YC?Fg=!p?#`@O`y>9xZkJt6CRk^Rd_e24p@74uJtP&OofxTSWX|e zU$tu?j^?hr4MLAWZOw6Ez9Ee(f!c+~rF@&Z@Vg#8VN^Xr`Vyf1s06<8fMA-acxc#^ zDc_$%ea*5_rfew#I51tK9dcYGp+SVqlx#|2o>x7&$QxA&)nJG!(ATmWHSo>L6+aw( z)kIG&R-MFROjXZi(Pv3R9f_?c+{)!z{M`d)2%Ao=IAgAd= zOF>k9%0igCP3yY4U;5(Z2~Dh+i=`&&Bcw-^D(Q)tA?xX6d@<+4w&kWrpHvCT3#h9Rs|7c-yazGbN>B&>mAG!Qc5U-as6?-X;lAOAku_f4?31sQ$qWwL2(Nu>d*$ZtFY_;i0$b(~DA zBy{Tzua(V27#eO-3cl_orr>+@jTuF>y_28Jncy!o7M=ar<3JK6c|}NLSfl_sO_H5a zC9ld*Lp6j+o9fJI#qDPVX0m6T$`r1SV$Y>HeaBx2@^#tbjp@tIgQy{UP?H6O%EN1a zX2ApYBdzh-a5ea{`^iq1-|nC8Hw%Dj1 zg*+{A3WSLTVY5l9%#?jbHs4IR0&M(H+O2cR!-LkmuvGVKe^Z8Kp0XQ{<^NI?a@ni@ zfzij}S}r4@*^~&^c?*IemvX3e# z)FynvN0{J<-WM0|bLE_stY>@kPmXy1l|B-H!)K)xT*Pk(B!l(A4(pNfLGORY3PBS(gMA-=z4;BaVH+&Ft!7gP_4< zwb`$0KD@)h>|NbcObZK53Dw{Oi^x!)=z{>&8_n9C1g%Lfe=+uhrZ7#8Cgw{2plpNn zVsdZ*gF)`kt2zl{_y&b5w!DjzK7co(Md=7 zr6=dl0P7nYZTOv799B@uP5L)Ikl9H^g1NBIxeLu>P{~7?DF1PpxXNeP+R+n9@&os7 zkWHyw@skPd`P-mcVbDZXt60Dlw&$FVu=11PGoX1PNL&+IP+=F7GofvF{O;8eUzRLA z41AORZ)QIw)&j$Fp+jm|XWL3N4koWmQXhvaY+Y1Ol3X&8p9cOmprwYs z_nPITPn4;>L&A08JM~?h{(WV_gD&DMn-9BlDV0Xfcd$AZaIGX!SkNLTCEYuqPhPE}NUQ5IAOPFKl#ttFiP z;XvfcyTY$&sJ{zOH=g&`EIHzdY7ExXmQ~1OzyBQ@vW|ivY6`l_erK#w{6i1r84&yo zNHA2uKwk^RxU}<=S1!7e1+6&<)DG*cjW^+CC*DDKUzX{uIsTqtLxgZsJsL>$8DMNg zl_Jm_4*l*o0mU!UNv5;3yq5ioEzLumxX9volAFi zNfzS`6C<&@$eFSZvT{?^ZtSt?+b*3%g;ly#_TGuSS~t^B;iRS6p$f}IZi3KrK_<{yxEw^_y|mm#94jr*n()^(d^?<)69?C% zZV<|8?buXM%LQ)k$x4l*TcpjBk3(UR5<3#^w7jII!d0sSkQd2Wtel2TBJLxTM3Zt{ac7|>&xVf+w7I>W^ zjI(|8EjE8lC$=b$-42Sn;4-f?*29wYcGW28?LiP*=rj$L@S#D0h~OVQ8N;uxeu0Fa zuu{3q21v!n{IYx}KUKGUbyYQR!(r>URK6$#TN0Tz#{&fo12$x=&IPX)AvR~Fj`!y5 zd-i%jbQ_-&LnULW-ial-mgL@7`*9DSR)P$6pOEHICp_#A+!#AQ;og^JKc=do!8DD6 zi-9c2qU*zO@!?5eUBoL)+ODDfQL9JQ2LBFx@SzhJF@~ENw!nhI-16?voktGN$K$R! zoU)QAg(G3}(2aTGU#!-6xtlZbf!4r|W}vcj77Luy!U8wr$>6k3b?8M670Qr=AOidL z4m!5}yuW&%7G?W_cYkLq>6F5qPF zwKgssF80~e@OF&nhW9sRI80cufAq^`4hlx;toT|gj@;+&G30`{4Pos^R}v^Rs$DV6H_L@>WN_na>{!h_w#1=&AT@1$wnYgPZ$ z>h&sIEjX~u;R>j*HV=`eRg?WFhIdKA-=M_!i7O!_c(Jc4_eUia(pItn%4IHA9a$Na zKEs2uM^u34(1V2L#E%-BOWw>`NrC?UQdnZqboq}8L*!%0atZxp@kBL4w+0$yl{&*$ z^EXvrjtOT1g)X&Y(=WNiHL-gizK!goA<~h_BjF0wKQ((QoVtp(bMDF$^9p>F-9g>x zmw)$WY-u*49ULNOk{g1QP!7Htg=038W3e0guZU_lX|Y$rZdE^$SAKuG@_pq%!q~j!J4@-`X!&2nMLZ!QSn%l%N?km1`djk=Y zkBa{`y{|W}q6r^d9-s9rY&DkS4aKX*R3MoXcD`doe?`{sZOkPqe7?p3| zmA~66xc!`ROI+tbbuODd*c`KAai-9k|0GaWxWAXYt~w<<&p&-D)xbfFC^|EYn(EfL zN#Q+exQ})!4z#y1IXT}=ZK`*w)f@WfS}A-b;F9h~uF*YFcGBkO!95B*Res_!(&_z| z#gnzjdWyrR(i=yKg6$QAU5%AytQN>jSh+H9=CUSNWOioAuCyHd=P9Vrb#ytdFRJnY zD%jwMdz!J&)IQc5sA!9z0Sb>^5+;=_khjc7Mz?=~be&g3#hz#ZqX@E&b)-liSCFA3 zqa9!fh3Pe#0Djc?6yv{gcL@$^$J&Xy&}TsM$*pTpD`)^}{`nC1HJB*aT2Y7?4{6$N zWBn3ZToE26-6{}d`9$il=ZaXrPW&CP;zx$)tHK`1?|cTFVNUlw@=y<^bS|IaPQL_U z!B2G%_g~@}eUEDiBKd>g7BnqH2#R7H#41mR@f#)M{}pL_Y$F&bWLG z%Jh&nHYQhtVM$9^b!cMDm|@FftkYGHXM#obcKtcs>??z&8*V1NHGercd);&0AY7-i zWGo&G0*kRh?%NjsYGJROUXfRt#)xoXW z6MW9-jqZ+f5Hv5cSzIn~o!Bu!d|E#S{69m*yS+8x(emuEDJ@^Tw9c`Z%AJ3}+I77w zjvxKS0^C(ZtT%b9W~s$%k)dHBigNE&!4cPZ^vM__$b`n-uY72{Bo_CKMK{YY(K!dB&f6e0#brl%k((%N;tr2-M}Y2&syp5577S1q19Ip5{TK;|!1^@GQ82 zEHDZiR(2a`H9E&Y-(-Y}HV=6c_>frI3z{Y5U^*H!U?-R0vhWDFIwRrevicy76zy^Y z52Oppt(myyd&i*E#|-3o63E1Xx(im?Gg*#0J2L26exiM~N(@->1II zKD)yTzwTuQn5%lpjruJ#D8r3gr#<{iN%%?PMo&G$2b=1?^e zAP0*ZWzgj>7o8%lZhb8EsE=O_>tUvJF_1OaKm3eT6xGUFV)#|O|2UHSiY4_WgjLlrt-acU)a+8+YZu6W-%y=?nLgu7HN{A#im2SeNY8K*~5*lCo~A=CdE-DF#6QBB^%mkwP`m} zq=jF+tkQn*Cj@s}e!Ky*@1IRo-{6Nqg(viU~f8QL<$^uN2&=Hiz42Cprn zrns@jOa-~QLYom^EW2o)Ea>AFbVW{v3_6;}9BFIiHfEU~M*@#lcW<=nY;wq`c&05T z+DV4_kWhyg=#bD4w!_vZVQztJV*=#+4Ow@e`50Dx*=#D%z^`2ahftIERMKMEMk`~o zv;-k86JPV@_&K)BCOeF-OU!rJOv1Go-96f=5bL#z#Y-_lDjA1Xj{bRmkL+3Bzy6_$ z`dhw9@Tg?J$_k&hSr|WM`uP|I^^x=Xz_N&W7o&iH|AdQUnt8A;a+LSdEww=>k}hln ziRx6rJ6cn)wBbs;Yy;3{D$MdA#+VW}?B$>34=mr44P&8^I(|~!RcU@5%2z2>@r&Wt)f(_t3295q)l9&UgUa^_rM9=7sXXm^-y_oC^&sQ21?nww{jouz zCHXK_5VyywpZrVg+tu+$%7KV&+xY;M>eXjJ{R2+I<9?LD@1;Vqr;H~;!@olP>EDz2 zECyPo$dUKwA~ZHV{irzUGJYooR;k)Fb8BwW6}QzoNL=~J^QBUh+<88-*CS>wq3|d< zR#^+D&d+-um3NmfDlbkyXpf8-y-WXlqsJZ^38m+$umO zMXr;wk`WYeZ;|rzva)O%?*7wopI7J4;bLR&>VcWr??B6%;ok%Vk@fIzLboi+6nI=B zrgmi4#i}IQPjVj@OVM8vhsAd~0*iffj$4ghC_}VP|E8UkaWb$`KSmgvZ0dsybT&>$ zCw|8x`s^mX#Up>#iQ#)U*Ys1Z(>w!sv>)Dj-c&D&FwD!5Q-(=fDK2`78%?l&6d>Bo z0R2dWwbESTuRH^mnQFy-%s3Lusd#@lkudo|cnAx|qb+m6}D#Q2&}4b>}L3m4l6zX2C4$A z6LyB-LZYS6N`i&=S9c#vlV-tg*{-0@8V~siee+L|LwX6OB;x1O>8!HJPv&Ku))891 zh$6|sp|4uOU2LkphnF$fy&YKm!`2H#XUudmjySWpAS1yAKP?%saz&4TjYZSVXBXlh z;;eQkBR#b~SWdX)Mjjsnyg{h~{STz@o|H(dDzCYevA3^%tIt5n%bA~cp<)SMgt@y< z0zWd}M2`x%C96EmT=cjLaWgKq#tPq3Dr6gU7P23XH46k}RE%O}F*iErI7r`@&D#H- zZUs_B$5r=Js-iKYxb+bZ&POK$g$D@*Z3r0~8+v^N1dE*Kt6!Bo?90{nF~sQ)@iTai zxM&3~bx0hPP_k&G*oe{kv!@VyNyf8VU6G@Mru{inugV zd@1TX*dPur%w-NyeQx+0Wdnj?E+S7R&&6m%)6%qiS@JzgpEK~<2`tI6k>>pEDCbY< zh`b0;nduJ!r_6fDju^bYKEY(6&*>F$ybMOEXuO{m#G-$MJ0Bd~*=G3Z=CcD~vqb*= z?2jR&Yu(*_ot&8pA5OU)dw8pUo-G;nFC<4Y(!Do-Rdwh!Y;jz^; zl71K=QS_9)cWRNzfa@F-lr0#!Ls?eAMGi{z`WHjL7=vI`?DOrDBET7uNX7E%b~ryS z-d?fNlz~>dHirY91ajOED^=t6`9*F>k2Igm?BP<|sM-y2!N5Cm_9lx2iq{(&$X$!@ zjHG_xzG}A#X~zko{LbURlEN`6T$U;?mDF&`M_s*|K$tF=VT7FjkI$=M0sbiYi=i9G zgyTSU(oit!^LgK zR~~v}qdB=Dc9?G4Cr`gY-I>j6n5hQ)SX31+0{@8ZU}#I0IcZM*hUamcTrw9f-CS$d zI-ZU7Wh4hH-kWf8_9kFGv5WkTW6)+n*P{{Ua@JOLl47NMr}v$r+)vPv_^Z+d(qod6 zt6_~(ra~w2F+F{XIepG<(30a;1;$d5o`NP+e~qcUndE>ZM;j&j`+5^>-%yoLVUkWg z!OdUM_w*UCSK?8y=SoS)1LjTiqtAo$i;>@xW9nZr)cV=V4#h2aY&AI%F8C}lw9Kf` z&A`6nbaGOdU~_4;4M<{qp%f$KHPiQ+ZgM`N=i>(l|S3<(rNy# zx$K5g6_P^~yQ|~4y7d*ie7$ikt4f6NRP-%h=oNf3_HXc}5Q3w-6joWH+Njaza;{U!>tbc%oqFxqacd5?uVD>9RX>DpFK8iL%+xB z1m|=qe^hA@_7G#@WhdDN+6CV$SGwWX%h?f_VkIQ(E)|W#lo2&Hj!byOY0leKBhrd9 zwuchzGmOdMMN)R)`YRs6%Nd?_88lBBiO=2eX8_63SA!3BuCOMB+V?e9nf00T(_=1Q z#nMZhEExTa&JFKLIW96qcSnc38LZYxFeEEF)*NG-i4s;F#A$RRCRDkjkNn$~E5Pdk zdcWF#EZ%tK>F-Ot_KfG*uI&|!%^FkzW%|^AD^<1u%?8dm9B!Gate>%e>%zN*+#|GE z-Ne#AI#8m;v(AV6QCzpG@ADT9)p;kf*mCC4N5-IfoN~0 z#Z&m}L{IKMYmOW-%hS2r@r2wW9Oj7tXXF%!)3Bl%Xs&`q15MD~E)}mV%$+`N@1xv% zVa$EGE=urt>U2Lcu!n#0(}IOsUxa1W)Cu#G^x^-iF!6u$42bj7dLgmLp8+`>w)Hd@ zk&6;@PxO@CX5khK+UW&z$lnl86pt{yStFWJyVsy3FWw;vo|?hEA87CU0GeCoRp-+Y z{*+K#LD!`vA9kz&^jKF_NmO^xTXFT(HeLOOhF|LP2Zxp;GqS0qUpF$MZ`ty9mTD{b z`Xx?p+lvaCEVGVu!^J2ENU3W7{CqojSzJ74`Bj!cPuAun|5Nn3VN-jO5GG5CwJiCv z_@kueGax|s!%6vj7jWG6*16gfx%cA-7_`vUP^`L9_*J1cYMH!3zdRM`sl<`Kxa1Wx ztvQp-iy6zxjt>Tj?gTR2INFJ;)Z{%h!~!6)TUeTvP|Lz?wX$y@?- zv3yK?v8+z~E&Fds9UZJ<>^37Z0zHJ3-VgghVXg9O`i!+9dC`)MIsl50uih}+usI78 zdsn4?g-@v?V(E=#@$xRKKlv9q{hHd%%g*jGE{5Hd45L_A*$OCZyp$|$B;a$^DBd06 zt?C}D^)2@Hem?1YAP&etW93BIe)^ZWK$r{4?^+a*PT^jOto&Vln1b)So6qVAD%0z|f zu2hM);r%C?^8*GQ>}Z854B7qc&blYMQ3?q(I0^(QPMg3tzB3(pTdYNZCmxNiwop=r zN7sy^^KCVDSD`_BwYSk=K5JG$DTi5XZ(+!7wC_4U+!c4wS(YxCDMH^YL!M$nmWG-> zKUQ}KEHb(nC$WA8jB$~}+5g)?P#1*NPus=|_T-%Pi2rD*>R~b6fp$*#2aPUR^oBEi zv5S@_BH*w}9y0`Px!%!cK3eQQZj<9})0d)%JDc%yFdAQTbj9yUU@Om#&V`N#C*vy{ zyk*adMI*b9z)w6&qLa`8=(a$-eydBLb2Fj%gJ{W5Ouy*ER7&zS%mTx0CUtdJs8H-- zXfFHbu?PAb<*pObJ_WrkNK++SNA{v?YNu5`6e1zr}-4{#L}_K=fW{4{___(&3XBEXOif&lJ(QxOv3!RA_x={ zBFYZIQ=Hj%kKrRIMFgRy7;C=}@4woch29N2t0;|(-lii0Y&t84yq%Gs1`MWW_q^VJ z%10$cD=xmx7?0gz>FmR~m*GaUxT z3?N}px5p4C|Ky=#^ErMM>aB^s-Dcp_M29=nie zpx#Vba5HqaFCa2`=a^nNcGtVSs3V;8e;nI9P65>OImhumnhafmIP2SAJyw-$62`B1VjsbNmI zSTh}qr`W#&^mHnCZ7MJ41Q#Fw&HYA}KDh%Yyd^O}33VIU5&=!rFr6%3Uho)l{or7V z;JUOmEOhI`9b&rA5{n|$-Yx@j`RTtH&%p&FBBx`qtbk#omPypm$@#iYWxA82OyU_o zMEs=NT$Gh?g>nS9WcLyC-tw5bk0-}87-Eq_7@2K;5nXNObiv5y$V;wE z0uFP_Dq=5?fAV2?zq6cWSO2H-p|(G#lefLxSPj_*<4*n&n=?;)HA?>P^6iN|Fxua* zO}G*?o~SLuW;-S{$mZx{N4WegB3+ME#U-8dTT;^9MBKSW8?Ri3e>cW>>zDv02OGAI z_s2OS5xE6H*ML5-<+7&VYSuTRnzauM^Q|oc=dQ4CEg%l}?AP?6=2J`~*8l8dq;ES$ zS5;Vrpo9z))x3dpOHviy>oOA>xJw*6(Z+*90OJ5LywO#?G4QU#DVyrMJ+O5I=B~L5 z$#(D{EXFLtm%V*;aAIukW-r9aUG*39K&r)LVueND6BE^Y#4wy~!2XqwGuldAMzTa2 zD2GtpS1;*x(rjPJ!iw55``uc(R5}eV!(>{`sINiH2Xqn=62ko4of`Y;YC~zdUW?E| z8?&?0%E~QNR(WaTC}w6$@Pjzwm-MMsg{P0u7upo(ZHjfOTe{2&Z{%(X>hQy?4~=zm z42oreo+s$clW4bEgKRFS`9RD*#5?&lMe-k{dF-}2g2N&VDh~&q zLD;sXn6!D6KPtg&r_0M=kUo%_uTGvzGC$B2XsIi;{~L!#Qno6Glb-` zMJ$_$hYIHwylA?46f5zFfptlaIYb*?h0O4}Pgoch|9hc=#{U(z6uW6G+ky4s_gKiO z5~FjQ{L}jv(yvy^(sB$8=}niCq?-QU?mp7M=}({DgHV|m1xxD-OTJtOBkHCcjSyG1 zSq-2vyw7jUxur)#jrnl5)+)9Z{*$vV2ye)Ig8S>tY91xBCmKBZ=^3CjiA>%jAv7(n z`98|ed)L-*qYiQi!I`D*_D0LN*A?TlL7oUE0V%lKwN$=JC32u!_&^^$#eg3?Ls=Rd z^-CH9m3Q$ZC%smF>?TMz;lq@~OoU~a_8U)wA+d&PuCi+`@}cfQowtJW;CSfRQ87PgegnNFC$Dbt7wTHl=U z9A(NasVW+u1Dg;;7+M=<+RECt*6+wj-X+9LGy-MLuJyLnn*K?;yrYO2%6$L>Gju;T zZP;g%+<0}K1VnZvSB-qQt*~(K#_5bwPCOC4<=pWgdMlg+{vmptXg z@o}b8^8qJie~&)^9ft_U^9TOL5AKn+fpEdMPlhcIKCkyQYfTwotrmNBt_ zcFIXF_T@JP)PKeFM%;5f?`z9L{7AYFIvqsycsI5J5@8a0`FO!v>d zB~KFArJ_D5%#K`%Us8>iHe#<-e+EnwV&(tyAC1cgxo5;!7{YELDdagrJGOdk_dE40gYtj z1yGU=8pw^dgd7&$P6O`TVe|X$Ee(q^SQncC``OTOjj)uTqvbK=1+n$hCRz~@>J0e2 z!W(3@(XxVB#UD_2;Iv$wBj2{wmlt%fyl|aFjXrudU%^&Q)R#3?zg$_*%ekZ@MIZe$ zGNjme*ngZmg5(3uKb0!5J-+s7D&dQ$(Bvwwb`s24mSxGJsoL_Ja)uwMHdtlzEtMS? zoB4U_o#VoE7mwBc>e@p^Qtzf*FMr8IzP(R~4C^x}G0dv_Wsr24+?*l-?`qTFI+kU5 zs=rTX_wH)53BNA1kcPySHgtQHPx()9SQzZtV;&$(-TqL@|0#q9nbdE=-NH+WT!m7B zY#yRJYx$ZMnww?m9<7R)dqNS)Mgk#m48oBVKAX&)-q-x{K|PVLpYA2f2O`wRG5Div=ipTZ`;swyt5U4&IB=K-+C% zTA8)8GeiCwzn^4tf}C+gV%jmztk#k^Y?HLzQzx7qFFQ$|0Y6vMKkCc{Ees#vO5ITC zFb9Cp|6$C1ARN z_d0j04w_z7ninJ%m&j1}Bnd$5=QJ)GZ`&;0qzBbH-PGFs5 zrxQ}oz66Gyjp*eL5>QTAagDVHb%&)7ChpSP0I)5cUByR;<(d59v4Kgyyz!0#g822A z@sD5q%bO_&iuKfYCL=+gmJ`#GBKeOR+CE&IyWyJM^b1iHU;`P>CUvNX}TU^${#^r@Trmpv) z%WTiF#iD5_{uC{SARFqO&pJsDZ>O1yb$rRKq3{OPQ(iNxyf8b(gtDSqeM7_V`sg_n zy}lXaf;ukgDm>xb-j}Kwr55Ove0)Cz7LLzFpMqONffQ{HMEfoOMFBth<>>bm zWt){@$JN)j_}eTUhdvhRbtfMenNF^tt(KVMeGf?X(pAh^-pi6rW4O(@;1*K|a6v>U;#Meo1#+JpHiR?S zD_VBS)(KIS`zG9O!QNyH$~T;pt|?{rR%|2>ZL9-9n%b@ZFmy>BjEG)A-!Ce&L#7Oq zBIBkJ)sXJ5Zqk3+IMxMy_Sk_N8@OQIS39u@(`}M5{dZGF(uFBbftMQcF5jg8z=;_@ zi0ts`uYJkgE}d-D8%zDE>wMo;s1dflA+^_0lD7g(6w7c{Aq|LgmDYr zX04pD*1INw-rLhS0tg{1zKLx0<+GZa_4&FB!uJgsW^@L~L1azB+UPV z4F#=#HH8=|P*(F{AF5@TrL3o$&lO++yHz&wKkdem~0zNDz)Y3eEX z(g(1~(6!?VEvj!w`u-`YUo$%EC0+jHZUV$=ZzDR0rjz_YKQ^t%6;^?w&n-ugQR6Uh z_2GE`rCbP_6ty1lO+>@qJmA%;P~c%N5;HQ5Mdga9j(>xI-Vcc)0f)I8>$^3FcEvo) zS9-2UyEhYdd6~Ab+31o5d~~!zPp4HoM=p|b2098BWP00`K%4B|^w6n)CcJNEwqP7s z`G9x#@Xw7&mD#}#BwAVFeEGJ#CT}OGImN9u#8K5>vW3~;slZ7vY(;lA3T_ZV>V3=R1FIGqc>V11c8z| zry-l#DwOX#|t79dPtocK-sVrGlo79#KY}%}4Nq!ls2qmK1cW{QsHsFKF61>%aXF@u4~e zc*2DMdcB1L?^Z#q|3>;OA5p|%yLpL=^>bPqF@gECluf z$xl$y9WE>K8P}9LeDZ90{!)rwqIJ|U%+HKy(^3@}Y#2(s5&g{`X=#O4g@yj;V(j&k zpK!XB!z;gw=G%x8JyL7|MKq9?z345NnOW3>tLj`k$Ba3J>^N@z{Dqd*oI@BY4$$}r z4q(B^bA)N2e;;^Rl6%J&?)d&ddA-?{72`=CvI5hDAK6AjkI1$=)4vRpd~`@)1BQVL z?I>y7v=Cp;7TGX?xNA%`x!`vLgoPKS7FsBz^5%oQqPS_{9fV$MBUxYSS~xrx`Iu9F z5;Hjdy`;DR!Q9}w4T0gxLc{M69NqF{Js72>fLE2$guUMMJ;`&6P@*y#L2x7})m6pU zFT)ox26Z44DSVwaVb8r-*YcRG70LGDebb09((W|OE(330TNG?i#4^|yZtO2Gl|E(j z?tKG{_Fc}OD7Uf7pl5)3rTM*35(5O8S59`Y)dK!#A3v$mCtE9ybNn}uwl;p3p^Cvk z#?n(WRyQH(a>x6s(q+mzeO{&o&T*J!_=96fZq3nk#iegm>sMpqaqhX#I?bkvC?28+ z&UGZ9ILqZYYhdTnn1@OoOo1^@Fj(|W8NYuntn?Ev(Y_{HFkKj=25mrO56JezN^>Hc zp1y_O6ook?V=axj_Tj%XuYaEbd5D-0&>0sv`|=KSVStD2@C#d}sdYJDzpGdjr z4J|<`&3#P(0G39>N@a@;Ydrj&S&;jlxQ+6_`;M)RsK4eXnm$vT40u(jy6}bh0PlRY zVQ-^-3^%D2zJrD$I8iDYKlMdOa;5diD6@xfK0vfAU`XaVK^a<>=SrB0Ol*yTwy$IZ z2GpU0Yr%{WCfs0DVSXeqac9WmWSlx}4kma$F?V;aEWB^!LlJ60LTMI+tp!9%8EuP1 z+TT~QGD)!+)A5oF48PtRzj8`B-w7&YEYeZ)bLS7f92OgBaNz5i3#5!EteZMaiAPiJ z_S1b=us3a>ytim5U-xS>i99hWfJEI?;%ANE*<2rEO?U{Lmr#V{Fk;i6x@3OwWsdL% z#KS=R$@m#1z(Hz-xXdITlHa$9&?9Vs|2PE=<-`RL_0weLGe8J5*i0BVKkQY;f4G(S zDi+Wl27TlB4y#md6jhcWtJsS5>{`HRzy%jL?r6Y4y4X7sc5PoRZWft6MG7m?li5aYm;hoKl&iB~u<=~T z`~&1-)QNZLXMU6h-g(-NGTgwDOX$~!Ra4c*$D2mb2P`Sk|eRB$86Z{|8ZT84y>`gbhP! zp@jl1(&CG|`vQv~7#yn*Qv7$5vrcc~TAo6hBD20pRZ&*E zP=6HZ^zB|mrkfzb%J@tB(|lo*>2p8I{9$RNIEXF zZVTji$!o*SEPRG+Z`Q{cSB-4s1?J)=7sL7#%120Bl4B+oGKj7sq=aP8K9n>Oyl@{z zyl^qZ)8;$>9|92ffX^_L(I$RnIc;TAF?}K+K+Bb8X~)4ArAVx$D4%LBNKzN6LX?nn z!>!b|eNKeXsbWA;HCE@Sgu1{aNLFQwD0&vrxRYnZc}i2vEDxRJ>!q`dJkNmW8SluT zH=dJd91jE&^oin8v)H5NncjQy2k2N(u_ff!h^?HF|761 z>I(j``Adkd za(1%1x@ynAoxm?vg5+rL_kmWyAt|GkS~o^xjJas15eXET`UaId(LF)%C3mQ2F8FHYuSEF7IQe@Zp){p&$84Vg z`iy|2r>$eDP8tab8bczLF9zRr|9fw}Kd zGH0EGrA%}0i@$Y|Yy}Xfn_D8jHB5O;Ag(k9W)58N5F`|#?11B`|SbL()eor-~Oha z(bI;b{5)J;f#*hURIGY;RP1p&4PL+d_1?eX&vx^HxPx1I$P;$~ZXcFSuc^gZ!hxlU zV9M;-E>!Hhx6cMUvENC-bMFhkhGqzi7M6vDk1T<_GpT+H2;9oWM6ki01XiTZngnE> zO-TNjX=QX9X5QsJH&~@_S@C_~GSDu>yh1?9#bKk5hm57||CQcf^HSO#7f)w>UvZ!4 zj+>+EvP+gQTxflpsm zI6DP+n)e0uwr?i4XsyfmY^Nr;7^hNqT!}(?x2h=8<@=XYL>~hQ7pSPya0*6+y_xA} zfYj@wwdIUf5SZMWEfGoLSml{+OS4u1f-6m+wme(%F2}4x;Y6!~y=?jgvU zZUM76bKUo670DR4L0;`RjP5QQKm5T>QMM%Wzm-8iHJ|a}6MBS71kmAC5|JvTAExri z6b)nlg!IlZ#Nf}{+zB+~NWtzjl1$hC-zhpAoTG;4Ahbg9h#Kjc?uoHI>Pj&?6QFAS zgHMXe^8!ug%t;zZc~AKtf)0%0!{=~E=yg-g@4F%sb-^8SsVdBoZ&tr9za|;xokgk5@+jll^JPs96Nr2U6XF>Nu0wf?mqf8L?v+=4 z|=Vvt)!9xSVolBdllQTirYRA#*U^B8R8zw zq1h!O9;0FSrB;Dp{AbaxNs7DXgULm~NrbRp? zgMcd08tLCI^vu5`PLvSTnn)eBoE`=xAs$tK@3=g5KkWy95fWV**Ay@-)Mmzt?Sp*Z z)_r6uWUs}*hlq6FzPV~{?XBDE)z!|DC!am7V9kUFnyCrI?uy5o&2}>IcYd&Te8`c3 zEl?;U{q`u1LSo*=r&#UWHZa-n{|^4@0ik6{G6`E&I4B6Op5mn-Da_1#%cA4XeN3MM z5+&506`FqWC(|2wc$UOUU|$ns>oy${f>KwkbRRnT#)wZH7U+=~B#rp@j7;@yam&Kw z0DtkM0de~`CHS*CdJ0MMCoY=YqmI^8?FF>GADr(yVx;W__(CgrG)`NAIDD*U=*;4_ z+|;KW;fpWkKHs@uk)kP|2iYZu7~-XY2cCjS)`KJ%u*>&%E8z(&cHO4Y(AoqCmYgnv zuomLksTB+?jpTjhV(V{SRasTz4Vn64R55rzBcHZv+d+s4N;&Gy8mdCW;0#`=Uy4O~ z^esZV+zNhhSr3y9LeVbZtRs6?pBq{Xj-mc8$*5lOXN#+x-1|_5m;1fLm(-D^6}bTO>Vo;raq5Moh8$D zu<6|>dRJ^=G@2dF+Y@I(0=U;R3B$S@d%E<2eJ$!IPzArioJ2;ktdQmag`q~(M}dFv&>im*TRVOy;ZCV{EgXH9rDAczTfw__GQ48~ zLv!#ueuX12bRSjC?78G}uBYsT&M`Xo|Ats7>wkUxMS&7&iLbX%A{b!C+fQFCLX&$e z<%WkB+}6SH8NSjl3Q0l>ow=tb2R^yo;A*4z{Tm5-Es9bePQ&;MpO(hM#!YgwQ69!W znU3knQR<9meg0`nNL_mWA%u#JRk&@$^u{53&n|x*AM1-rGchNQ5#$syv+f?J?=`ZBLZ)W zsUspk3EG%vRtxvsp9@z%eSJjxhjt<_J)1K&rIJ8Si|{-ba%Kmks*YGU+76VvZ@y6Z!FUnomX0G88%g(a@5c)Jx=62j(olvOLTS){KCZFdQF+*Ev z54}s_k?+3$5Vq+3ipb!v8D7i2kfx=6eOq_!-OP^bV6h*3K^J<6G*n6TbxIpyE+EWF zDi8{+Zua4s84#L;MTSp`R`nANHR$9+07){O_T4q;LdE zpLpAX9RSSG)N7>Vu}ZZ-|CVND^O?jo|6q~8VnxWGXxxz|^{jVq+)L!ay|C`|S^a_} zUmfLoRaF3F^&;22W!byQ`>!5_*Ij(vEeeiRlvP4#?bu!r7kXy0xaBMIwS)F^Cw(Go zU+P9FaN%0@??EcQ>-YnZ#)gk2?}RnMrgPe{*6vUX!(_yRdqLU7qP z_WplTDtqd%I{;S`3zl$E%{Du77jZ8FY3siTA+EGiTpUaBj*lNAHQvCd(aKk&$?2_L zvT4H^02Z5IF4-~KGtn1%VaOu7&AxtnWBCsOTXLTM9&&03x>Rwa#8F ziqX8fp1-~UHsJ-_F{@^P0NCN>wxt@#;e)H~#i%$1z^W5z1rEg8vV6+yJ$9MUeAZcE zJcXtR=^Xxtu&H5bQ8oVv$30cR)ou1Evn^ul@D3>DdO(VSWh^kjyi7d7?)ijzq}Y>% zaNFD|bT{w+kX}>t{yr zZ@}DubagrBI_G)WsE%XlSp%TPa*?3nr*S&(NN@vRJjW_mnLT#z$C&+iH^3mN{@~jq zf?rZg;0}-f`&;kT;+71QZFCo zFkc3%;HN%B{~=tepEe+Fp=_YT1@iw@d_sUL)PvIZ{}66Sg5f$o`M-{A3X=ff9eMX3 z!dvi^{{|2)-ikF^q+Ym0g$?29Twa(jC_*pl-k}H{aSxwLF&|^$WDWm!u~kc?exc1D zN4)hg%UYoiA8aK(4FyFUa=rLiqs#Mn33s zU4To$#v$P3a>joMku?@24g%)(fh(DHD14{Ce;(TWgSW~>Dxl``an8wq%h!JLmH&S& zlm`gN|9N)=Cs1NL_JgM~#+v+3PK`<^P%8j3OrH9Up=hD1@Kc=HYEG6^x9YVEMer@x zaRcJje4ErY1DQ~=RU`Cgm_B$XRuR+fY$A-eeTm&c3a8Jhao>F%X`zK=zmw1jKV`;K z&;7*LT1dH>iQ{LzZV%%X8&|v=ox*bj?;uQcOsA=KccN%%0a##_KcLqaC-ea%`12N? z$PXuaFw=7mPItqClAmg$W@nYv2N1E}2*eg~ib0EPZR%K>^wiQ-?Nk#cy$aH8i_P@Q zq@@Oc2z327=u2KV2gpd8=KN`{r03(f)3Mk=0qmpGlhneG3!L ztdh@biv;I>F-fJc>s<~rsV?=W&9^D>7i!Uy*YB8=6&uL6eSv)9EtaT)qolEAmX!yq ztaWt4H{Uy0X-BG~s`QwI#A@JDUEh&(R`KJab!e+PFwK%W&|6!|W@4->79k}Q?ie@+UxR4%o_Sf+Qk`40&NWE7#y)O`Ge7BJZJ{ zsvC&3A(+Bb0zP0_ZlmB84>EszbBUU3tqqY&j~-1N0MC+7?B~8AwUhC9Hnm$iI=GMIm z?`V#K`?94eZ>ekDxueOSa%3?h3@|}lrLixFojrdX!so6zJf?yL;{jjWDZQ+}`u{_J z!@V9ey_94t@9Dm^nEH27D*(q!iiCHb!=tyT7q|%1V?t%UE$1?0u+m@u=R^M4 z2&0m0>^+p;hYwjB(~Ek)-X82PYBFiU^GGAz*<7#$6B{0)Hu;YWVT}vq$yOL&-*Ia_ zI~uYXm3=O(pc^8+LdUq9nq1D&@we52)bT#`{o6DIT9Hs)79Bo=Lg)JzlYtW$DBF4o z$<+MKZxqVG2b1|3hO=Kb&w_I95@Qbye&+l4s8Z^}l9aM~bKm=kEfQVM8@-L*@R@{# zq^Qp9UrNN-6V2FP(hcO7l8}LK-2O$uI#rYINLs&eiW=P`kvnOdjpa0K6_wY7| z%>dt~WGFP1*3$H9Xpom)(NKY*Ma|BqBuSli`VQ*u4qAHn*Bi<1N;&u(^M<>9Q9jh3 zxev2)o_HEd`LQUpDR|Kdxa5wC7k<=j{5|xJcBn zw}S5e^*wPdK4t9qWIA9zo*F1`$|jk|h&(z;y@Ux>c{XRN>o6U}DkR~lNw3|uxV|#u zSj3;ItC?mJT%U@Hd6I{wA~-Vyzs~QxHCKCihFXDVnr`HPcRQx6tXN z(A{#ggOS^fSN8k9D@A#~hSmpNHu7ksND>r9ceE%U$7~y7lrp;V-i56wU~nA5zk7yB zOV$3vtq0Ad-a=^fai)#W0z;Oj#3>x5BsA6~&*h76khY%>0rj5hcwfWcK;Fl@SO z+FE1sr=Y;a5PavW^uO=*5B0dVa>!|O(ekBF(hcrb($_u#T;zC|yam<5jZgR=Vf(OH z7;}<>r%9&T?MKhg#TARXUbv{%u*m12wTrv(l};JDEp^wGOA9J1V`HSEW9YA6653>( z`VZlNGza>>{!$Ri2%7WW;F`7*sm$zG$Zp4?_rBGuhXdpQ7d5Pnv$o4ke8@4}F3PPDJ(qLqm6kR~Z%fxK{_*(RvP zDCTLv%RN5s#(RE_=icMWFO21LO?sU@_;3lgIaiCT@Zp*Ymq%cqqvA3JkAdVA+O|DL z=la~X(8V`+%}X?^$gMKv2+Xg7gKadC4U*4IL;f2yFARgHHKKvzn!jV6c_Pb}NhZIw z6wBYUP`r55&;_dw;OeFe){tyHtt*t?iMc|56dw} zJm9R3Io#T&v9k1lvPfruo2eyk8baK<`wr%6AQ2rJlQe3uZpy4>JV*Vo^nU`0xK zRdP|?!f83R@iXW;+p64&Vzp~`=3kSduq4+^7PnAV6dlskK2GJr#{M_wSnyCrV}kBt z@|*r2l(7)k<4AiGF6woNdWMrNrI{i?Ml<%6!j3N;Qld~fKa2e}A)27c7EcOgwu*Di zLcupu^R!ihSX9Novc+-|$T=jN;#6oh!nU0tOX;V)Mo4;-TQxLA9FM&=8}W{6``pRu zvoDnF?vB_hxeAuiDE=-^EKFY#GDySuj0~_hmgjOvXpnEzuZ>6kSXylXurMxsN1p-Q z$}m?883&5v@8c2&bEX8izP~Bc$zs1(uD3VY?M)ay1J5d6Jz6b3#Havu6WCcOjCNRF zZ@}|?j60@rf{7ej9Gnyuc!!!Ayxn^;Nm%d{CIKh#O`P42-)B#%bsxwh^e_<1da_fC zjH{JRL9KIF)yQn)gH#nCLh}qHK3Q!g5Do4oQ(V>XOpg!b zc-~dGp&rgskiLpLsC6m-6-sk(HJwyW>DRFxqD^j%ZaHgIki4q4mV5Z+l)^+(yVHvZ zVEo;)25{ieyXqQ2J>t3XzxZ^wkW_eLYv{Syh*bn@?-9*mbY@;iLecYY;P;B03ff!M zY`m^?{b&~s)QI?F3Ti%Lz#E@Qx7D31>sJqsOx{N*#%KTM`b+L|viYUo_92i_#~eEM z!v}*|-Q&mX{?iq2ja`hHy1{~g_GXFvRf&tmisCLbdgE!{GoNi&8TDfqX7YA+2+39} zi!;QTcB8yY?5R*xKYa?)L>K`0E|e~DE5IUP~}YM}S zisM+?P`lLWB~JVnuOo16XWw!^AFvTSdR57WQ=rZ<)vK0BQ8qP6vr>KL@bB2CLj9(D zwDV96o)?UkO8)1yOVd+pM40phHTU_phFgtBhL5R}+l#)^bi9G#&T?6j(BoZ*&9|scR>P|a;GxP-&_yuXVf4XS70;Xlp7e2lDqzINtJ+b=;VZ@iZbc55@qcP4T}4i zEaGCk*J1&x+uTN(#@RaZ%9(aDS|J-hdhCGUAcbu0zUeC6V?0u;9VJSxsNEf@Pi$bl72@7CX#&tf>qdAH#VJpz-tzAFS@Byr?vP&S z6r?GdFlFTb{>YCwJNvfsZI#IA6D9F$D7Zl9&;ifL%mQ#sMbim$2Kv(-ma;UAPL1M* zh~gozBhEg>UPF}lAT4#cvz0UddtJpA40Pp0crPT!B5b|@Ok#80c~ ztcgbA#&XM2FQy@=JEi_9=aBWR&|c1yqW(8XNg_@_Rayo7w>`Z^yqBz5s7*2$v?by+s5giG3)ffTz-l3~1$V<#9rmm;5@sVuv+AoV1ze(K zS!(q8&3;l^1P!JyUJOK7SAnBbzRP}q!R?)W#3hVTm7S@?($^y6(Bd|nu5PysSrSaS zS1_Dn$Sb?Trx+K4lpHAioJ#^c?^m(Srx`|9dLF0!BzEaD9h`}{QZifpPoYEBI|Y;Q zbzlN)mUsBmH_4W;o-?M<{IhucJJYW^d+(!}$36@o+*Yc(gZ@>B%8V1qP1ot??f9&{ zxxz}dOHIgcCqzKUS`+)R;a$3F&dz6%tf?aqM9`7$zcid1;EtC@Ni6MEU4-w37h<3Y zp5WZ4I&rh*GcaSEWOHr?+9%61H1FHe`MZ`I>Y(&i&LJyfQj(8lHeHrYcQ|AA=9;s% zFLg7(!MwZDYBTQ$@j@bI&-=oxG@okSv*Tx~KL9 zyYab{l(8u#R+)21r7O^9$DrZ9U==S$f4fCN&d-q> zV)-<{PhbVLWOsvVrtW~eNMPylLp8On5ZXP4xY^Hm7Q8|_0dimYE#_uhxM@}}Jo<&3 zKD4I48-x3xv^~|^$B}7u+hI7BDsCJRM$24`(@7vFD*|dhiajDPR9&*?fypAV8;Ply zrM(c6dHd0EGiO6{a{*mUIraOH?WCa+Cp`k%g5N?OsPQ7Z*%aeM1u+Kt%f@tUb9))c zH~Loy9Mjfx48(JLE1YBwitDADX{!&SvCi6US&mle4z2FOU_3*ToKX33Tc^H5!jh3f z0zbTJ%NBuRx!-0Tz`bg@RGT+f9>&vQ8JaaXfe)X<&OyYGD zs0$8wE+Hv`HP6+kd9Mz}_L&Fpi2m?K)Otxu!gl;BGzu6G;8-%+CxH_eB%r^`wq9AM>pl@?)1Oq4-^00x7L`(<1Soj$`Kaf;KF9dn5@HdV??B+eF(z zj+6LZIE_EeZzZLR|KSFnYGtIZ8oh@IQ@#TYFi@F!d%!stS;+znR#%AcZ$4^z&WDARplbF38(lVk_pU-!4A%ZI5s5g80g?wx}Q6{rRjeP2Slu~ZB4q(h&#%4kHz`eI_>G+4MlxK zzs?pk>OGd^ZUmva6ucK^7OHa@#=?tcA>WvgdW_9|;ZR*Va$H)YXkc(ULk%Aq#@{cd zCvxqmcJ)Jdu}O+T>zBCFr#F{}R7$GuA(~iKU7ZyYoSmJsdmP>~80-?$^)7|uK;6fTo{y)-tVL%xXYvZ_j;MnMHVO|8OME?XyDNQK(}+?X zD*IIvnZhMG(uTjfjxtGJ1Jmpb6ZXV=|FozMY&n6@r_#H;(P0xdza6~{lu8H`KX+Xw zL>`6$L1XqZ0Q6;tz6o}PDXSbyMmBx=#IG#4n#CdCzm^9f}I6b5(=9S^{ za513nW++JymQ{TF0wvulqX2Rg%-A~NKy~;;OV!79S=S@HtwYwn7um|c8%D%wiQJvK z%(B*V7qW<{E10RTGR|Ck3gE1b?Mz8ELN$M3GtV3W1~ zs$VVwB#o?izdn3!tXO{*;!xlqm>AzIRfs6{xn|+i)9m`KL)^ zhMbFzRYojfEhDE7Ke^0(?J#jq;WsqJbjaoBS=jLqH$SQCD;(-@5e6a!50XyfSVfeP z=wo=DN)0G_NvE>!JJ=~O6w^LNp7J=A(EP(WzbvmEkz*2O8vRPP8TRZ*0e zsh|iptZ>9gBtgujs1-BtN#{&y(SRr} z@fO)Hiwz#%yDqJS|NdR+yAqI^N`8zsO~@G}^sJETAG<2L7-(W@Y1SwI@dcl;uclMx zUf!zeWtE?ODBe&`w2YS-fzk0lILz7DbNTv3*3rxrHX}v!wS~wCvhhajY*4#hgCSX} zHRz7_abXOB-_up#>r3pW%pyG(f2Qlr_Os}ussZ{5S&qGUl7~J1iOXG@G`4Uq>iGgDhSm0q5yMSsym*(37L{HngAwZ}#|c<%=u`QG?(!{kWh zg|@M%OoyMK{>PrScJ1gl4I5~5{NU{Yy*~M_c^!m?#a(#OhUDi3AW}@%}(4*ObgkKz-_1YUj3Ksb+`gq&3EE>FH$`a6_W+3Hb#H7$$ zYuH8YEVEBT()d_yU*Ll|e^Vy>kd9QXsGP3j^SE|8$Y68bJrA`DQg#)b>1DxXr!Tl= zYq#qV^;twYqh`Fp!v2b@Qa|5#p$NOx>fITm@MjSsUu#P(5rVjhv1MYd&we;9agW-G zm&17c+~-1G5@)`kqn9UBcSbxJ9Z-oE#-|&A3{C=}Jk|=IcyZsB(K+X2vjwyPf%ipT zG+EcyrKcR1N9IKB;!6^)W4zp6jr82ox+lWywu9r9D+vr@mNMeOW24kZf!hNO9v0H6 zv4m&_rO$tvC_eqWu2rc*{uyafMR7l#c2PLEqnniwtLte5d;hYxZYXQV=}CxS6Rfu(Qur+%@QD5apKS(jl6- zlW^J7_B(;Mz}LjlcK@!`M7at&EYKnA_O{IlPs0i;O#Uk=3B z7^2IZ66IdF1{0hVtE@Kn+|9eO>5iNHkl} zi~c3{W(Ikd-#ZXmK_oDX=?j#0DnhM}T-e3Zv(loC0Gq%A>+FU<-tM0(S%pF}HQI;I z9QoLT+oc`35_iq?*e^Ub7Jj$Y&Xc1#{LxE2;?3)KF?-f$;{5TYDGPa-8+hVbYRBnG z*1I}JtZ3sz5A+GYCz;_y&L;nmZKURU`f^Ncd!BhRlKXZ?x9mnW*dB4Ky%B~;B*ej^ z;$I|Ao%x&2GsD}Yj8kidWZac>y6CD}`|c8wq)o-ZfV(Y#8kWKT`!)-tT*P@o!3)Ib zGkKb4?+-xlrl{i0o3C6U5oScmJ)pZFeF6De|Cq=Ox$@wB0WU>o@_;`-M&JKJRWzXA zff5NgVeUvHb{$7`hfQq0t3l2GA%L^eIUZBK^+=`oR6<$YjOl|{ z%Xpic8Br>fn00Tz?WuawEHieq?e96i7`}q7{zNXztVZXYdrjwAaW-%&8`TF;zZf~+ zR^l^LroQ?r%DDgTNbs)P9Y6X;rX9Ob-v}KgN?1~Mmg**-)0{z?lzW_lWoCUuC9gQi0rYXVq^( z&8~;pQ!TOa`Y&~n?I@qjQWy@^^rvqxFGsY~ohiznrDv*u_zd8BP@ktYzsoBSCyQfX z(fkbRm(IKd*xi2f9hq{B7E*sHL!emX)Y0!1t?G{)!k}L^Mj2O3x71)YZ=j2l+}35m z8a+Q%TtOYAaRgsVnZ}kB*~?oZdu)|Sa{xu`oR1x;|D~D`gV%3!CXNYSO86ZQb-X#4 z-C{m8{+}1D7`UGR^YtdTuqycyacA*@6u!D}R4;=R@)lIGF?04UEoJQC$cP|LaSgum9MV48RSC{spTMYPx{8ht14NIU{XiQFQ? z7L)PTq z&kOPl3h$r8S^q=$Zm<6|q{(V+Ys%=>oGYN|=)G)V7mzZGkq;`03m}<5Rs>1cC zT0Ax8aXSITFI@Q{UHinH!`1F&ue?8b6{dnQKjGl_XYM&_z4C5>_H$^2xoXe#5Fne5 zd+7vfRRqy3lBC+G8;eaIVp_u-v;@eEd}phi((3769K#d{E@<>XT2j0EZF6YMD-zF> zSQdqIP@#B98f=MinRngk^#o2D(Y6nbIU#y#SLMh5{F4u?A0*lvOo8&dfmi=#%OLpei{Syd$X%tX2BSCBWhY@sZ%N;UoA^R?9j ztuXnRJ)Sf&^`tSJjrKjzGNVAvPT*zu>5!y;*v8?-5QJkiZ|&bI zm{fax9g`LH{X2>Um&}m@tqlb>0U7G4@kHN+JJI#22G+3MX|fRKfM3$iI=And)oAji zxb8EsNc2eivWuo^dtBPFXl(^ES(|%1+5!*@1`mz6?7Z=&G7(b$K!lzvw8g}$=u+NS zskZFRMMW+Bg2SvI1? zy^d^f<}1BIs-@D8;DnhRf_)%yxqJpo)mY3&bz|N^o$AmOJONeqy5<_n!JzTSxmDR3 zyaHvtirw>_CS0Rl@*PO_oOaTfFqyC?u*l?QD+ZB2rq3joat_MRauMlv zt*<@E^-DYn*%^?uk=d$}SoJf;@dM@$jW{>!;UTQhc7@90RL8iAK2sARcb9R4uR zI*ABFcc%VidS@5X{|eT#(JAwd%DV@oo(Tq(L@0m%*m7USDJNV7@hHOt9nePn9z>6R znl&Snq)s6>627C}6TtxwNF0=VO;dROhPKpB@zuWfHgzc(<-KKdOJ2=l2|P<)u$Va; z85rx{dJKH*>c&SQsusenCfMnS=Uf=y6|j!`USQI+n^Vl7RX^D`Wv%)TpbV?6O+MRF zhkV5LsUwLxHlQ%OWKn5s`XWs}O8EIJpJ!C8!pAD2^aDp)l68!VpDxDnE3S#YFU-zC zCAga@omed9^3XH!LAHVSj!kkTr-Pv9Y4;_cU&Rk4eaF^WbzQ3>?EHdaNR5)00Fm?y z?^lDPmq5-O^2|c(p0e9}!Mzx*IpZcS*Hhn`Ju0-|A(cVK+*@P+KmSU0qpf9uG-!;_ zmS8vb0;)+pOfMW2rqO(+72L43iR~TLuD545!`=?Vrs?Y4c>bt)$O_z4(EJSfZVA78 zM0y0&mq0(P)t4d{D77Kq{B%CxkEtUg4k|7x!SfHM$X9w*Bv?As%ztC-s*;K#+1Q793A)b9b)J?Q@Cj*LAx`mIaRUvVU# zgsCkiV+azVS-6dE64L+Csck(UP@i_lj0)64mNTQDnofE9s$T3qPxfL~J>Mkow zI=RROxGY|KBE9#H{6n~~6$iBp2s>b=*nm4+YTMiBN=f`oh$5ln%)zr&4D%= zCCh(NOh65B#qu} zn6bQx2lu1<#sTxXQ;YoHqg*w@%Ox&b1f(i;0m92TRBTt(VUJqXxMWSPkfkp#zs*zk z|6n5V-d5Icd1oY3U(FY4lG00KwX)c)$}_NCEye20ePh!9Y{b!>%-i+6gaG)_;J-a@ zpw$j^r{NgJfwb+!zY|{1Y5O&}7=2rzA+tbM29nxgAWVr#tI}_x2D0%e{t2{?lt)6EClTpV<&yYSOKr={$S$QZDsv;)-H7!q|+4 zVZM&9rpc=!!T4_x=8J4WKqHl!CVekv&*g)oTuZaIe8(mlnexsMAd~K(>y7uNTxqP$ za%$A{3=s`mPO|L2Bs_U#3(u#|oqwB3!9$=5iAn44T7T)U)Bs9;LKKRX3R=WqW{wu; zSOYMV_o`%kn4+|+N1z@78AFB_tw2z15+#jP-L99P?HMy9cOo|SU2hspR z!&S4Cj;a#ek%dOf)*fQz&eHwd!^+KYn@C_DUvICdUL0FhQ9K^hUhZbZ_NXDigC$8z zHbgw(DHP)gVSFxEkDrlRZl1kG6E+X}Nl((Zke%ieMuk<1>fMOWWm2qWLgnJbYy>J@G8l*sRL=wXoeE<-H%yD~9%F=AXsCo?ELa#{F*5F8wafs~?M&IgOb8j4_0Y zG}-zYahohyr%y?tlO6U)ZfxOladbA%CXvl^ElD&?Ma7<8g>UWgDNSGhpi zK<_kO)~fWlLI(G@p>kH68o_y}VTp+6SSAB?y=9!{FImMXY zjRZ!t;AWxPtDfl@4D^;bTNBk?YR^E4tQS~w}=E_-GZ3#h{BJW z^nxcO!4vVG91S(dS^Xy~gmg!kI!fgdhLQ1Slx#*}6u}y}?7W8xJx6h!S!WSJ#CU`M`TuUTCNF5w`|&!1E{PK=m_#v` z@ZJ@d&+j=@e;_qHrOHzGkX^=U(DEjKEXN8*+@5;iQFyYfD#Mom=cp*Qg9a_AuQaA{ zPF+f=SP~R!*=+>%?L&BqoKUN&ZRfzrE{bw<3Ov)3;3qBi4XC$cT&{Nim2u=c$d%h9 zmo8gGvD*_Fu*4Ue6b10I#+Zf)%wp11)EanRXF4m7#f!#GAD#+{XNfAzTXSb_wO|Qnw`m8rzKFwbfy)+VHKsi&{TYEs5G*_iNZOKZw3OFuB zr6*CNwnZ!GGvCsF8GB4#y&SU!4KlhDX!vTud7&59B}TRSWRQ`8U2jRr3WDm#lJsbG z+V(VQe?0MW^sbdR)s#&eiF1@TJ&&x=kY-Xt7_9`A=k}MB-kD}6+P@tIO60eq7v6H_ zu;q8V@EakO9Lui^M6vGucp%A1lB_NL{g`5GwBugt_B+4(^iyAmd#T{hzYyhT(G@xP z7CBLlOrAmpF3YVC?xp`dp&+SKNOfB@GI)y3xd|&fqa1uF^CQ*5C#uKnG10l7fboJ7#ww$mwXYFsrNI>+~4P&Y7&Ivo%fN zJPUtN^e9thD66O3OV!|yBTy_3_(#c7vVIfRH}S7uM3gWbyDDm{s!M&2Iki|Y$GJ*r zte*#S=DgvV%LV!I!FqKRKPXtOK2mvZxN;Ui_!sQtvmA;(@)t^3th8aDlN7@(Z?s{M zwN??11$}&S#4?sW3Is;g(dY?wDjzD5pj#*V8BrvFLI3i@eiZM;`n&QCpBjG*WR8Wp zT9o3AV_Y^dtN=%%6h(}>>ilbLTheVgBrl)7!TW)2?vx!nd%oRxuz-%MML}JvVl$BF z3S@0YM**BFVVaf!I$r(mmvzLmn{NmIDaq1h4uL^Jtsvlbh}tR0TAgx|#Kla0MT~<0 zqA@NN&%7p9zJx6t;P0aBX#Jkqr^mwnaUZ|yMe#xN>zz8!^fxBv1%VHlz|&AJG5x=i zSF2?(!zW`_#{=E67qFg^1Wf9NuT8hDbYA=a0;ND&zu=WY{x)0#T&xc|xNI;a@_lJ- z)aZ?A%SOKgxw_NM)ElLBbk2GDZ8qIB3a6{KgGCrdy4Fvu04&Ac`G5 z06SQo0X){;xui*Pq9wy$-OP+KBAtYRewi5`ooJ$#sOFsN)Vw)Xhn`DppqFU@_BD0w zZA5}Uon;4{o()o&|7Y$>)%JRe0rVM=B0ualx#bXzZt26&Jm^Az`$)Tvc_K5jgcVXu?k9V(WQ3l>1_* zw6ys$%1LUpnae38I^I3)xv~#oiqMWCtA$J+=Bt##QzKOb9i(wmJ+rzMU{!O;9jVxa z$cmK*G7rj+uqrT7D;$jEo|IUtp$xF1{LT4O#e_iznTKQCaZ6EMT&VZMf6D3r^#-My z3soC-0N%KzsiNiVSC;YS1y)4^*$oFU{m!1$YRXA;B8~!2BvQTGWbkPXCOpJfp0!z} zc{t4laZwR#2*$l*s*McJE>L1NGMn;%NK6X&M`)|JR@ikf_Q<=gYC z2B>8)| z`A2%5V%%DG>Q&Zm64^Yj^^%eD1Hc~S*E6c?CVRJ%J=n!_cXw>jK>>pLBG^DNw7&2mUVwK>5506l8CI2h?p z#avCj2|2|hao4Yl&Nvjsu+6F{3QtO8+$!8Qx#g){BCOe3ILnd88Li7pFvM+K`+Cym zFWIv>Egiamqp?Et{%zT zLk`!$@@f&gTZRQb^0#lsy;nkl8)#YfkfeOd2U2}2SX-&ZSK3ABc!r~*UFq|Ab4JR2 z#d}__q*-cLOJLEX&AST2IRqS#$FM8>slHNSIonitL>975aImV9&zuj#HRyxF@x!jP z&=B!jD`r8!E=~X*-D@edk-To%=Q2SgN*^C7uEys`D`2KHj4(1bX8Ws&I<|WSVt)#b zWZ9*TFh8wtPhqFMv{OZ>#deapjFWC*DA>+3pH7tSJD95FdKxaRn$m5gxro!H+`N-f(KfU$!tT^O&9i)hr6WLUM zTGhVOo;g^vCth%PU#YG-ke4=x=rF5BQ8`PvE&m|&d;++^z zRE}~OZ)B4Z?~@%s8LjDy81}TASF*1I@TTNt^=7qfmrtC>gt$}(&=5)ds(^$D76C6=k&OJxF`uWcYU+{QTZgzR^Ro;R?UH{E)UMGLF!A`LzY~;_)<~F? zd~?**nP#|^*!du1uobeNhm}^08_=pEa=;ea4^BBWrftj{VaL{~G;+$4=)_kj#YA}> z$u!}v-_8YeTKtYpf z9jQ&*x^7YR&uT-ox*^fSGDb$y?f%KlH6yuj3}tiJ6GnxovKN`R6~M^q2&*r5a{%)0 zt)rDaBuwM!=}DE#ZiPU~-+43iIIEGSmvRTsC5xWtGDrUa#eYg#l#(l?P&Yq1`vID~ z+C}718R1mBfOVGL!-4h3L+wGTita@m3Dh|JK@`-GX9t2i9@G%xT3C;$KSN9oJ!+G& zlesUJM`KYLaC+4lw#$}^8dJNV89AuzWG^uXpcynX8k-vFf%6f8*i|`haq^C*r7M_n zwpwXS(gah$>S}<#g47cu@Hsx_mB+~X8?Yg85qLpdjqi$XWz2TN%8%E%skqe%v$m30 z;vY1Msyg#tXFcuR&l1e!dS;c%SXb4Z+jx%5T_vr#GRWTT8g^QF)EC%`3A{ zt3e~F@lK}F+{2}#;5i;?2e+kj7jwyTsJ+=gGd4%HBH>CgUS_X|wWDyUr`wM%+7Jc@ z@ex=|$IQ!eY8kZ}FowNE&%R0PnoZd~Iikj_et$Y?G+08G#(inDSOg}KxL>URHIHG^ zl7JL0Y3eC}sTn;g)CGMiTd`>g90b+))EbUvYM_db9>nR*UwAjK1EnrtaO`9V%Nzk( zYX;ucT;%&crCui}h-nkb?f`BjegKKS}piXwHXBcZyykQ=e?jdxjDN=p^95&>ed;~5#ob3;RgSUr(>u4D%}Z)1=@A`kSabhvJ>?gKa{oRdiI zr)G8f9o^f74J)ik_qoeAJqP&Jn`mwAAU79Hwk5_@MoW8UrO2AFr}mKBjcuAs_@|Oa zx@iGTxY`G<7X*xZ))uckdbY0}&o=6-<$mElVxu_YxyZ(Pb6M)oE=nnz7dnlyz|g{~ zBY+mp09P3M-=Y0_S1vBv-Pm2vCK0!!U|>jBfOFV;b)%8DsIHRG+tu{huQb7?3z(pl ztlDYbY=I7B+(QwQouKjD4)sFUQj1)Lt)0)7Sji(saKISIBh(D>p858wO3Z^C*m!qO z^E9EV&3pEQ21ynzu}2{2LuWmBIRdAi-^2Fp0kgc4{#JyI?;^$%frVndPi!CXt71kr zp#sfq6m7k3V_z;*lc;qZ5$YI!BTtjpSN{{T`cS?*eD)@M98HDdDJ zpngFfwPYqSeze3jCaIcMKQ`q(J5-u_sztqyHyW08(Or5LlH9+rsYJ)3AwKlcvq_Ij zbV2H924?GlRm?ADyAp*Ii@1H`nmLN(@)XccI@9J8rif%twDXF!CSxiwWeH(8%KlYm zMaLtpQkyxahKZZz6+Cch*8m@#e;Tyza!Q+~?u)k9o?ydp!m{@jIR_OkH##ZfCmkiJ zHSPt=W1;4tcV;;!CYn|-tu<17$sXsa>2pg6xRucqeb~nUeFbb>MYb|u_mcYdU!`j( zUqibMnkcmshPa49%dk)PhFJXtORL+9ONNqCB#`nDo)_J}!nq|A=;>BHh%Mz^d{jvbcHo7RL&BIQ8}wi_>D=p6~S!I03fV3~Em;)!Xs@f`E1| z-9g@f}C!>_o1mJ z@7hT}dC2Ia1M;lx_;pq3oVf$^p+qm*e zbRXxnGdWXI(VWD(eAe-V5$Dy*@WoA9}{fFyF!^8Km&j%yNYR5+o&TW`qr^W1y|V=qPu8k1v&xK)}s*} zU+fe!ej3<(1~;AW-MPgHhO#>p;OtD1M#j~^gpXSFPM zZz6I_400(eqUNbpT4-koPWBI}{0(WR6Z2TyW#i`Wmg#l%oS8JBz(O z=W;x@^CxvBSbid|+_L#QnSRl14i4k{$WPbP-|*t3=W*;#y5+8*zFx6vjM({ZVO_32 z-UIJ{I?9&Jn9R1!W{7fdK4tok>Nuc_8aFl(#DZIUvE(=0k&aXGVbjy4W?J4&YTKcc z8T-n4R{Dwsc1=26Km$n`3H|luPvhJ1so;I%eS`g;ZmkA>NB;nH9zO&4QFcy&r(B)# z6M^#7CUuu^`#QxiIAWOq{vNrjcQTw)M7I~LTce8piKF|8==)OMo6b1I|^M=WbeU>=lMj&hK-TGLx%M+-%X=P|CWSrd8; zd0zYV9(OlnTn0|j%~p7pI^qe&qKZ;?N`I+|SPMLZ4n6HU|J;r1zJ?%4fJcU~XT zV~-h&G((}fjwNuy|FMzNw0r~LElT^5n7+Uh!F7nYHEYH}Jk7-eAF zSYy-?_|{ctQrEH1^|bVgGj)wG7EJE*z}C>a5`w0u`On+JH=N%Fp8MtJo9 z06w+L3X=A!H_@0%PNZ9Bn_b2&ZBj5)Ge^1E7xEuN$LmnsTr`&gYe>YF=&C+v+EvDV zeF(2n93yS5vdd=CnZ%4)BOGIC>OC`q_!^T>lI`xInj3iCfW})pR1EXSS^&orHlFfK z@t-E+%#5liUn@VteMcid<663fv}vZee=2nuA)-5snFDJQIt-K76t0-{E2_gi*X!B!MTX|RWSwhV}nfI7*!%$>;&hUg_xnoW8SGFCp&Iexm}OAeb(zvl<$nG>N-}40wi(E zNdEwJJ;3gL>!O;`ZS5Cu89dMQkPZt4;eV}5yBE6$cdI6am1Ain$qT>(s~*Cu-MdN!r` zyA+NNdB<{ntDKJgmPJ$tU_AP_)`b9A#I!n=wuWedj{tjjtXnN)KtvAVfIoY-d(h%* z98D>FM3-88oD31q6?#2R37u1egM!_v)zGAe44TVja^*FTjmR>*Z} zLt@hiBeyJ6GB-a%SrFd0o?+TrIXENJKmB?+S|wtmEfOY%<~eR2+XZ~ZAqIQ(AE>Ov zxCwh31}Z~;Ju^G7D-KEUul z0ZqjTR^@i+S@Q(rV&PaEp$G1_r_j__J7b+@MZ{CC_4FR!f#Q{iDOuUQ2g#e%+%mUu z{opv?j^o~#cTN709z?14@Oj7gzYg@$eF45@uA;$o9bIKm%HQh~``_&ke_BZ*`%;j=3MyQDb>p{i2KoS7H@IbH|Tw-G?TG6Go-hc+D|TID8r|}xHV$l z;G|*;h|I8q{;jwv{_mmo=}%HL942YPnN&1`k29eo(E9yp^4qLvC5BHn2RIGTgXl-C zQZtboG3cU{3M@#)k35l@vmAM%!faUL>BU-#F;2ue1aR@vti+)EDE(=qLy)msgV25^ zr}F@+u@_^~1@Fx_&jYEZiPW&MsO&1g+J~(@1!h{pilq0d2M08+K2}F>k zH3mBEM1YUs^s3K08kT7k(VcUp#@T4j)~jtUIXu*Iqg6R0A~z2&PB>BSY4XI*Q>fjpOD>}{w+1V+9Jk$&3~u+W3oRl)wL=kOJj}os zZr@6YIHXi#KBs%&-7oCgJZ$TSag+Gg{+$?t(NK(WUVP`w&Zw?cp(5IEHGa$oYLu=L zm5j^XEA1Ao41+4%k7Hd7v7&2rY$UtKC&M5Ui|e8ik_ zk5BQg$W@K?W;L->lzCFM%FlhQ!KB+wX*INBSizB|#C}s1&r{c&^X*e!Y8KZI1U8N5 z!yLPr-anM!9sxKU=Bg5#yqr!cy^@hXXx=O?p{TCy{o;v_ z822NS>_F+9yWC7r&1MWD6g!isR~89e<>VBA^Ol$Pev zFzVVp#8D{#F7;E`iml=Gk{NCyel|wIg=F-48u^^gy>Iqg9Xy^mar%nU@vI?qy*uqRFr2N*0E^jt4?6$rYkF=gjsXj_OFrU~`XA zOfI3Z(S2!{b2RrXe`wGB>~oroBn&g=b56)kq`Qxn?{WLYjQUWp!QXRJGje1fX^?bd zR(3+YBmC);c2Ps^I@JapU3_mJkyS=O~I5;YQK0ittBK=Ddfh-Se zjQ;@3NIU-kytM_p$+u|#09hM(k)M9!@uJ46Nvknqm7^;v95SAN&*@W2BQgn4e(GVk z4^L6}8U*ha4&GCw@Nl`~{gdfQ9@H$7a~q5|x1pdpl>C_Kwo3kDs;=&%(9;+goW|d$ ze|i%Jp3=~{Yk3=gqFu+nGyFcjy6-r6* znq%cBdyH0mvxONZmC8^oJ<;{Rtck8moG(gT#)_Gs_rP^($03~h)bkdpqfNC))bW#9 zf+qB-=VSJ9G#gf9_hbC(lA{jPb20lklSzMY-7p8WAc~hW8iMRJT}*npq1OYaG;=ZX zuES(@mm7~tX#8jdtSoVv8#(qha>CrojkJFjD;c{R(~mK7#aNLAy5*1g=C<^^4=HjN z1obt`I;kCzjg7AjX({&D*}i`)WtjISvfIQBr{B$Q8NrN!_25=bPDrHEJx1celF&~X z1$tw>dBOOFs9Q(FrqTW3nxA6Y*g354eGcjgZ!M(`u@brD5-W$({whf{!aIeGjQ;?4 z6rBkn`$)?})0Xl%;X5*~j!@Sx{ksLsOEd>-9-tb=borHmK^Ap3Yny3SHYp$*qdT#N z9FM}HNkrvkz$+ln_>a!GIb9i=G@7x6t=cTI8_S5^?m`tU?9iQzM?B}gc_+|UM>VOD zP=W^2$Rjz;bkN1PD;Z9blexj(wZ+s&`#^B03&RK8g!I}7;QI5>p0&LP+C{NTsC>Dh za*o7s{{Za|t#*BF#q3OQ@tWMLZK_8#szUNY@PUCiLC!lc>ZjkK>}v;Gu!=ShYZ9lL za6sS|?Nu1kt?wmfDJqnew?%zg>Pw50WooMoFgKei9bNwb9>XL7^*=|fwx$3ii&0q@ z!O3M`Xc;X&FYw`Omkz@{1y zX=G}bwwh&?pV{N{H#=u17{UC3CbS+!(xbSB!ZsR}#AFjAF8Gw^t~eb=2UFjQEpWeL z)YnUUtlV5cry<|T3NyrNagV9SD<4anZL~~;$0R7xZBhJN2nXx;e>zqrm0Ivk6!I)m zB%xPh%EnCK=c(`YtM40LZn(R&jwlo;YpDKIcG@s8{&}KLLaD2}8om^|xnOQ?n6btQ zK7y;-S=~;6#g>73A{=J97X1`O=40#9v$gx-t7{Y7r~0sVNB-#GS4BA43zwTy69gKya#OkO)la}f)x9nKgOZHRb%UL z^i*P!-I@0~ji~b+?%i%Mht!@k`R1~wiX*vMj$0V~E1Q_@ba`}N@aCSJ@Xn6Ff)}oP zR%MfUF;1WmOE2S$(5InKntB^rqYP$BxLq9^dBsk+OV)O{u4(sEl`{2SJ{)&t&3;=?Ji_LtIx3kOxuxb_<&J#kLx=5T2UpsTAd1Rg58UbpZl zA6hO6GW4Jz$09v0z_lNcT>Z}(JDiHjX`IqV3eqYv>hN@RwtZQ8;^)(aQh~z7~ zBRpf$nVX74{{U$FTpn2BNQPGj?+;JHp-DQa4pYSI-Lx@;$(;Uv;dsu)Ov z2>j#F_Y?-iVci%IO0u55l;-{7@BSqpH@Nkth;cO{zcbq_v|;ugv&GOdo{T*R^{mTz ze3_N{fx{0_dWvM4GQbg*NZ82({mC)NsOL=RX3C5B7t&`H(Rrc&N6HS3Y(Y6ixS;!`qdmXj4{cnaxFm)d0s>7?N(xi zV;OE(_NnGABr+RjVV2-lgch>R&Ni)0*-FTOM;Z)m$@Z(p7~xWTid>~Rq*3yjj|qyh z(_^5;S1Ar@ktL*uy-JgxS}egy4K|y4ntMySp|)!jnG0Jk4>Yd$6*`((3@|7Sj&VV6 zl$uGhoMU5WK(itZ&W8qKt}*vG7$@tHKN_W{-4+`qGD#XKk>ggubJKUeI5iWCniOTo z&gK{u6k`JwPQy?#*heGewgdA#SEyR*7U}ae1I)%q0|0-G zWhCCn)l}sjqiIvaq{Nf)Aj=xQltrAUU9#p)y!Try#tWByxWq2VlK_JXdu&rtIW$O+Dg>WSMm7lE%{LUFvhMmiG@Kc>6Z|E*KEF4S#%ok1e|XoBG7 zgpZOL&q0ySPp>^`^P`Db*>;=(k;fookH}R@_Rxsfv1GF(w+EQ+06XJeIT_0X+mJKI zr9}nIv0f$2Q<&xBGDZPzeo{_&&rY<&(Wz{!9nG@^xLJ^Hjcz4bqAXO5s^bH%_|=$J z4=*!andKY2l$UI6Av}9}ew5h>*_W>MSZ$-+OmRNX-fZA*$2k0Joz!$cI$T||#dgV; zK-pdR>IbH4B}dHZScI0sUHG2v{@v}8Qex;^xgM2=WAdvb3^JS%=xb=wiZq1z)}-(d z0Um~;Q;A6Rtx`JZvTb#-s{cx5|Lr8{%@ z8poau*)?;mR83sYWUYy`+X!^&CzDf%+>C664s%UOUcz=T$ic-)D)E}yEaR_FOQ8|U znqaMLna3H*FGAkn%TS|nP}Ppwb-C4;kMf++FG>*Su-0xW7jKrV;goF1OnGvq> zkfi>#gx%Tnlv2BqmwPI-Pq9eHb5Pt$$fZycN#%&F8FQYNDMc*aQ|)#oSRccm{l1U2!BAilwAODh;^@gIols%VHKs1F=(2l%MSjJz{O^ z5pOIw$n>stu4WIFncMy9wG)cIE;V#$*3$vI({&9jw8%0L;g_12+04zlXH_Shv8v`qE;6|2u3}MwnWrVCu)&h!C(#!) zxsh@9i>f9Xj39il=8!Fl9wU-N{?WxI&6MhYG8F2M(GgecN#~-U-N73Dd8P6fw4d{g zYpbb8QU|c5CTWfpS+m(zo`k+<)v^N|ta<6~YRJ=WpYJG7_t*+)qCJ%V0F{~=g@ksV z7PDixyoF|OPndC0n>Tw$j!VRHkp)EDNdbBR`qoigvb$!^683hnkdYS~l>(^h#L!fn zb*`Hn4rA4l6T2oy{V4(t^jL>BIsX8Gho&gbP^kKfWI0@f^)$GqA!9p%g}OQ8*wa{2 zDl)LdW{OPr91v=0K$a0O`|I+_@92M}2)kKbY)2{u!>8PCp5D~4NEayl#{GT0=oXLE zk|H*TC+6ryM#KT~`t_z`Bu-@H^HqbIE>9^WvfyH&AhD#`NfCfuy=uZP++bymNkgKH zMmQN`SM6W}4BWLV3gotSJC6A(*_X>-dMqws^GLC$mV+3oGfjc&O*DyeBA#8%j>4ky zo&NwDs}pgBiyR8I6}H|_TBM2QL@N^kS_uzASBiY1=O~pirsnHc9u<+f?b@N!HH~X- zXDYmnk~pi^5p0WUedgd)x@L;9Go)Zn#pfMr+*Y1mbIGjjt&Nngt0}8I4w9z~ha>zy z9+i2mcd1jxY4WmMmvc(PQfXiivmKEr5dw3B$sm3Nk@!|kqckoRn+zDME;?tBe?wfg zsVAw?LZ?Go=@xjM zQM^fY434QG^2z0UR<0A1xtv^5JrHZUCYFn9BsfGE3_6 zOVy%3+4r%RJr#}(c+kF*IXA^F#vKSar|hJ*W^Pb>qp{VmE^KdFHhf75!(eqE*11`= zdG5~X-*aOHiR)c3Zr<_5QkuWIb=JE3GBMqayyLgwT%Dwu;;zRu;bXglT$#XV-1O(3 zE1t5DuN4x6WVJ?o^Mm(U(v3%K{{Uzo?5~W@JptNG<{ABzCIO zoy3gsp7q;fhUJ$cSlJ*z1Fy`47bED$k=w6d!j&!_8v@0a1Vv!1xdvBm27aHH9<`jb zdK$x3c3YAgo8fHHNg6G;%q{K++bKNoJ#cVy?NHjYO+0^UwKB+IX#C>1!0IwM$@l(r z)96E8N_UY>Wvn6_N#HV!lw*|)dD^YqR;0F<7ZJs6bv&_&2Ix^UB7zS<+?Bx{s;^;K zp?hT-*&RwjJEsrHMeru3Ae?ozeU z?{5`u0{J%Va3z!YzJtA1y3{W93Cxm8#flio91W!X&k%rU+YrxxnyG8g2$7aH zl0pYK=~}jZHWC&|>|lWnlVF~agITmp>&o8=OHnNFjiYd0q%~Ka3&vv+l}}#O<79c2q~^9W-f+AEN&3_i z2+)v(lUpQ@Db7o%l3cTKo+`PjNf7GE?!|dBDl?iaE^C%78GhQE#zEups3LDJRCC#Z zsFE&}?KEDVLkL1)N8O%Vobx~!6t>Z#*hzIW zuI?KGwp28{q3jyfaFDSDfi%4qD^miXXQq8XdyE`#N0O$Ygpmp^|^!1 zs`P?K1r3vD7W!UKs}07xtyarc(}FeR%M2|bjhYdSnddxHR4FlkY2sd*BaS*oti$HF z@)!NDlnU%^d^XGCcr0}wZeW#5tbvFQF~O(o>7{f-#^Pk`UR3w4P zwPM2geQ+u?Pg7t zhxkbq=c;_NcRgQg&lJ&0IHtLnM+|KwjnF8~ZTN2TJ1FFo5VEsyPo)xe**Y z@FnHOCDa^25ag&l;;-L$u2~~0&y4VKQO`DXDsSkH|n8=Nz4{!p8bC7mYgs&}Gtx7RaxM%*}Pk(Bq%+{oPI4LBw3H2Exg_1wrvk&#EF|~e`LT@GRN|i9j!D2eEAJ&qL zek&Te&3mbY1zOSLcG!-<#4*%^*Xi3n^-g=RkCt4v0-~>B=~Uy*sWF>c782LWB1@P_ zcKeX`{{VRVe>$aea^*xj@{5pgc|Lr`#6*@=~uI*y>9z3MYjV`QSV zDmzOoik%K{SK71|$f;?Wvg0@dwHLA}MM_$gj2>OH6&u(Qlla!wn`{~%Ht9g~0W7D} zrC{rF%Uw>NtEtCL2UzMFdgzAMSrKF)t44nDZ>3O@&N;4`qD*9xJ%H&&p(%50T9rt{ z+1Xgw(XY%lEpGn+G?4t5^-_30o+_+XZjxKx0f?XXZhiWS+6gyxx#v}FN0dBtOuFoM z47*-f7kA-P_|DZ58#}g;wDLP4Udo_xU39w~Q#nqQ8q~~C*rpmlJiSLw;}x4U(?|p~ zH%d&TFq#b^j5Hbsd#0_Zz>MBd1p4Bk@ekObCL3rZk-M>~ld<)d9PVo$us%TH<%*}$JCm{ zz1I>=MQD1mD0L4wU{k6OI_x6L-<2C5HpAQN!5@WY$)xy>@E4mg0y#2B z5{W3cdzPg;TE+*DEMc?NB52qHGZrhj5(ls2%~J72g}tI9NvTD1a1SXHZVElkXH#P6 zD>rkAyurD0I%cDtw@Acv&18B|*q3e~jl+HN!24q#<4uU}wsr2T7=I%|dYj8w=qBt^ z$qbRivpNuu_fM@?GBR}JmD&N5P#F(uWZ0B8#>wbTQ^C|8-R3pqz z`1%?G#f5-0p2DM7Y<=udElDJWLE61Z>q4*xEYt?1a>cL?bCXYq&8iIV*-*Uszz-Vkkb&=5=>te6HqG-GF4}C8x$2{ z#`=L9J!ln+aq~n*nkANyQU<`LK`6yW8lAwD98^Y6N(ExTIjY`ape$F^t=3DJ_kGtG zKU#g^-*1ukknR=FS}m)iuMIUBa?4{H^+Z#ZkA)lxbc^P;PdTnt=6AK&sq;DRYj01| zjIt`K<(S|K9FheVMWxs@k^)w`O+FC{yC@kiKcU4`I;Fm3lHT4$ zK4ZvL`ZDJ^Q`;FFR51Qs? z^5S1YN$FJA@@!)XLwc)IHunQ09LMG`EPLZ0ndH@`Iu4b-F%+83$l;z+V}TEStJRO! zBe*r9b~&WErJ}KsCSoUYjJe&z0zXgwwN{!$R$zl_5U_Rs75>20HuYju$nmILO%=qiqPdy~hP-qL+e z2U>|72378QpF!(b)M-Y?MjkSAMY-qNp-JNy#ya}dR^M@b9k|BXRDqvf)y)WLbJD3- zmDqezDo9j=ms}5HSG1dClHOTlUz}wCJ&54bZ6|3E+@!A(blSF)*lzvoaEp(pKmBS* zjAHt0*JUPnl@r`|#cv5OnZZ^KmcAm!t7E6EX_Z1)upKfn&22fizVM!>b9BmDI|7Tl zvkd;Zu9!;9;gY%KC#5ZPn30-!6agM7`)Lf0oJ9aIic>(y$fY!fnHi-bkw>{}nwFnw zya_(lap(p|&?P!o_pZt^Kav?NW5S*c(s7@qV_n_c&mj^?kUK9*e5adJ7YOMU z`U!MnjfpaOuiW;lmIq6+vp19f0Ht0YBDWyyL;dDG{*`vI$0;f6qBG;2<3BR-=x9D= z#LVr>vB@<@E49gTjxtj}T8)m+mAg<%Y>sDMa5L@cPa7C~k@Mrc0DxC8{ zL#$oJ>^n%#@qpbknuKLYU~or6QzfBrmn@Og+G+ybW`rzZ_npnRoP%7$-Ad9k=jLOQ zX(YO#PI8si^g3%@S**2)=UuF0lXe@StQ(haJ)1oMHKop4lvB|3iM}GsCYz24j80(t&do!ZA1jYG5{f^&{}0If(wt zo>{7IFdw)#P%E+tV=<^3@-xzwd5eIh3Xpr!7Nk^w(*bjeYaG%T(sF3U10I6{oScD7 zmP;Atg{CH1jjAOy$tPphqs;;%ZnY-DX^dj=O3GBy0(5*-3izN+7uHIwkD7NT#m&`Q zd7xG&-B1@aTy`hlP;WF`EJ|2o*EIopE*2w>FecmRc%*j&<}HFd*EH!ahh-|=ByQ=K zQAKcw(K9nJ+RA?E&jkMfFUGNc)#W+=0G5jDf&n8q=t26{JenO2rn(&j7x1ezDY!5h zZ<{BP)N#dR!v)-vJ;-svEd|~;YM-Rr0E{s*y&YbFR7yehwOkw zwCx`;#tHh=;x!N|g~K#ze^36m>sZfori#(Bnt`+H+tQq?n1LMQB9k;*Tt@yLc7k^k zNrGc6PE?`V52Y=OPU_~Aaa`G`5$!J{9L5ejzp3@9qZc!Ib4e>SoNmhVz#fCwKb1q1 zT3i$(HG2sZ7b|Tf(kAplfVlaOeExN+%+TAb1S(lJjOX`+j8k!quO@R+#nhDfArmGj znrpZO*5^(RZ6*|G3V6`IO`@XTEHz^W1x zIy(*{RtG1mdeLrqUPYzs;bMIk zX8Dn&VQg8UO%0+n=-#Hmb9D)a-@4@{mcu3%>8#|tI zqkw9A*rK=8uI=sLd@)Yx8+xc>2kBj@-5gD7LyB5r8btzzMm1fn->?7jG`Vwd$+w( zlCkIe2+8wJ*%YO)TYGhBK{HABA{}w+Y4F`RmP@I>d2ruzJr~}mTj&>|%K3C%`jJg! z_Jt!4zG5=!2i&KS%>|;?;we4@r&yksqG0u6Zk`fzn8LCX$^yY zyqW%#D~p~T%(ZYEb1p#bS~6+2F#iCo)gxj$h}Sg7&l5bqTnq!+r@H$d#am$If#sXO z;Xu>BLcD5ZaG>%|1qi1(=toMIdlwdy)Os!?0G*^B{i;?1X54sUK5uhEOm4~OI@MN5 zHb~n)y4|U^Q(cq-tO~UNMJZvHjYra}xhmOLIH0=& zPb7dNfx*scha~PNG=YmrOp$SuN}_-l6(TNZu&yyiDt)6A#c`sX7J&*mz^BH%(-A61 zQ%1bgA+QmOl;ac%iQrQATy26?C*w(v%4Hzv)L6ic!*G)P%--#fFo9+(xL zj#M0DuR};g*2O_9KtY)s-+sQE=Qig12lYpbt zp2C6~AQ6NaEEM-7^r@!2wYAFH?75h5W|4Rw?ypRJb4A5w$)N`^Zj*aD5_^SF!2C`r z-)J%v&IpZgNL*!`ll-e0qpE{?v4n1jvUF}w9?$N z@+gtE$1=7FRVU^n)SM_j+4aRIYZygMOOPsJ0|sEDx3}er6k?sh%1r^g3qzxGYHh2(5n_TEsOweOlHpF4s|rHs9_@qYOVR5%|}uN=d68 zEl9M?8g6*4OvEOdO#mS@l!hfJpaiD@KoOBr5kT&G&x!A%wu8!vT~0cWmo+Z6KAmF- zONM)g1A`|uq|)louBY&xaBZf!a?wCQC+baW-D;P2Sz9;+5%*#-SUvPPB}SK4-^j>) zG{SdGz%lKfO89)Toy6mQY&*yLH7`EVp+%U{H~# zze-M8Cv;i{6#YYSB^KSNIiXbWyfCjE$!v zoz7lCg>W&><54k`$Xh(qAE5sLv6noYQY^AZ(>b7b2GiIdm2>r}7WAelojeJCfK#%5;f%tyJc7qpF} z`Bw@!1G^du=0L?YiN(%wEI3uB~^E9I?oT;QfR$6Te-&`)r9d7i?ceRg`D;1P`9&rvgd^!z3WZw zBbCpWz8Mr`#zrfeb=$iO0na_7pHg$bfnIAex1 z=?gM0+&x#Qz1DPt7MmIscJn^ZJ8Uq?{p2U8Jn`>b_0)H> zI?H)}^U3BMD1E93Jd6&4wxvp~EnKuXD%7V|Ug|I|t!^xCWVO19qMjqQ(Yk|OjmLt{ zysvqtPvqRHW=Lmp$sz5vk3Y`3;faK_Z&Q}09&yy+jY)R|wpWu`+yJvcR2AGLJp-&XVBT(n1CI_%yMRRwj7oID1U@|t1)ZH}};?_(vkfvqoMkq^;knBZNc2b`<&zPrDB}wb%d(#&v`K~Bt zL`xIb6sH;afd-ge3~b|*-mS@}$^DMnlJg2&t+_w#6)pUQCIv8QpwJ>!>rDoMG+9Qd z)YTDU(wd216kJv&%|*VLSdcYGx`44ZYAxJS3j?(=j%XOtd8ExC3Ml|lOkqe(0buAs z95Cod1XD?+pwno&y2bQr7tIoF>AYnAbwHffPK`Iv-B+r%lQvgVvAF>zi^^yF!}xPp zjmWH?8a)})9Cb~klQgBVX&Kr!4-h>EQB>_O9tYaQvK~FfOrbdJX&SVq^;RI9e(;a( z@y$MXi$xK9N2BCagPqSokzS1QN2=A?wFkPq(+3bbP-VzH?tO{_<5)kKj4ih3ww zbDzSH1~?2le=5+2M7)s|n8OzD;gV3S2dM|OF0iU>|h9a|2 zl(jnjtvKkBa_;IIRoaqyj!7qu{RLvnH<`5+c7goq@+G-G*}W1jSx(Yi%_E*l?(f+2 zq|+3w(lPs_;L$CKjoZ678a9-9Eht5nO{z%F!`JosMMPnm>QfwXyKg(=UsAj<{JE^# z^Rc~>dMlGd5x9{p?aKw+W0mvN{c3T(;Y=3rZ;ey;-dNxP)C$e$bZMb2ylu_8a?$ZS z1C9rA+aCQ*H_Q!)tg_Iy0F`(%p1XQ}wGCfZT6R`nfhsi9yCu9nQfB}Go#b~Xu=-W2 zJF928lEEfZG->mqoN7KuOhf428{_=gQTiCqX{-1>jqi?|K`xE)qNNn^N zfNv(}R-60M6AU^0c|U+PUss2J#LMe4doRG8YFhTQA}*2T+aNe55=mTf$>0ydu7;lW z5gf9JEff~=ZI{bT4#f2*A5W!RX;Y4<&Q)qlLrUJ)!(!QN{6i!+_kCDfsN4_Kaafny zRlU80iEhmd@wVjHGIQz2^{usQLj>ZVy`eq-0ERpz856+w(ZUW`Jfj%M7|uB4j%%K= z@m#i!Vbmd$3&U&!cjS1io2YcC!=pOAKf(H5x1)(#&OI*X;9}j9T!Vsku>|)WYdJhw zrOdf|^^EyqH4;MkMbF((pn>_+^H0oNsKL~9V{5M+9>J%+nMTvL0t-0KLX7gfed_G2 zd}pD-dj;cacH(Pr%M&fMz^GsWz}wH=AC+IVowg_0#^;?~LnW=$mbVHLc#hXi!0kSq pab4$$wcAe>MIDWejm)tJ^CY?usym*+6rP=Pn$k6Ukm>e6|JfDM1yKM1 literal 0 HcmV?d00001 diff --git a/modules/model_flux.py b/modules/model_flux.py index ce2c55f70..8d6a02ef6 100644 --- a/modules/model_flux.py +++ b/modules/model_flux.py @@ -34,7 +34,8 @@ def load_flux_quanto(checkpoint_info): with torch.device("meta"): transformer = diffusers.FluxTransformer2DModel.from_config(os.path.join(repo_path, "transformer", "config.json")).to(dtype=dtype) quanto.requantize(transformer, state_dict, quantization_map, device=torch.device("cpu")) - transformer.eval() + if shared.opts.diffusers_eval: + transformer.eval() if transformer.dtype != devices.dtype: try: transformer = transformer.to(dtype=devices.dtype) @@ -61,7 +62,8 @@ def load_flux_quanto(checkpoint_info): with torch.device("meta"): text_encoder_2 = transformers.T5EncoderModel(t5_config).to(dtype=dtype) quanto.requantize(text_encoder_2, state_dict, quantization_map, device=torch.device("cpu")) - text_encoder_2.eval() + if shared.opts.diffusers_eval: + text_encoder_2.eval() if text_encoder_2.dtype != devices.dtype: try: text_encoder_2 = text_encoder_2.to(dtype=devices.dtype) diff --git a/modules/model_omnigen.py b/modules/model_omnigen.py index 64c99ddd7..a08ad4ed5 100644 --- a/modules/model_omnigen.py +++ b/modules/model_omnigen.py @@ -17,7 +17,8 @@ def load_omnigen(checkpoint_info, diffusers_load_config={}): # pylint: disable=u pipe.separate_cfg_infer = True pipe.use_kv_cache = False pipe.model.to(device=devices.device, dtype=devices.dtype) - pipe.model.eval() + if shared.opts.diffusers_eval: + pipe.model.eval() pipe.vae.to(devices.device, dtype=devices.dtype) devices.torch_gc() diff --git a/modules/model_sana.py b/modules/model_sana.py new file mode 100644 index 000000000..06e6fe981 --- /dev/null +++ b/modules/model_sana.py @@ -0,0 +1,25 @@ +import diffusers + + +def load_sana(checkpoint_info, diffusers_load_config={}): + from modules import shared, sd_models, devices, modelloader, model_quant + modelloader.hf_login() + + repo_id = checkpoint_info if isinstance(checkpoint_info, str) else checkpoint_info.path + repo_id = sd_models.path_to_repo(repo_id) + + diffusers_load_config['variant'] = 'fp16' + diffusers_load_config['torch_dtype'] = devices.dtype + diffusers_load_config = model_quant.create_bnb_config(diffusers_load_config) + pipe = diffusers.SanaPAGPipeline.from_pretrained( + repo_id, + # pag_applied_layers=["transformer_blocks.8"], + cache_dir = shared.opts.diffusers_dir, + **diffusers_load_config, + ).to(devices.dtype) + if shared.opts.diffusers_eval: + pipe.text_encoder.eval() + pipe.transformer.eval() + + devices.torch_gc() + return pipe diff --git a/modules/model_te.py b/modules/model_te.py index 606cdb86d..8227ba3e8 100644 --- a/modules/model_te.py +++ b/modules/model_te.py @@ -52,7 +52,8 @@ def load_t5(name=None, cache_dir=None): if torch.is_floating_point(param) and not is_param_float8_e4m3fn: param = param.to(devices.dtype) set_module_tensor_to_device(t5, param_name, device=0, value=param) - t5.eval() + if shared.opts.diffusers_eval: + t5.eval() if t5.dtype != devices.dtype: try: t5 = t5.to(dtype=devices.dtype) diff --git a/modules/modeldata.py b/modules/modeldata.py index 604ff4623..4b7ec1776 100644 --- a/modules/modeldata.py +++ b/modules/modeldata.py @@ -3,6 +3,45 @@ from modules import shared, errors +def get_model_type(pipe): + name = pipe.__class__.__name__ + if not shared.native: + model_type = 'ldm' + elif "StableDiffusion3" in name: + model_type = 'sd3' + elif "StableDiffusionXL" in name: + model_type = 'sdxl' + elif "StableDiffusion" in name: + model_type = 'sd' + elif "LatentConsistencyModel" in name: + model_type = 'sd' # lcm is compatible with sd + elif "InstaFlowPipeline" in name: + model_type = 'sd' # instaflow is compatible with sd + elif "AnimateDiffPipeline" in name: + model_type = 'sd' # animatediff is compatible with sd + elif "Kandinsky" in name: + model_type = 'kandinsky' + elif "HunyuanDiT" in name: + model_type = 'hunyuandit' + elif "Cascade" in name: + model_type = 'sc' + elif "AuraFlow" in name: + model_type = 'auraflow' + elif "Flux" in name: + model_type = 'f1' + elif "Lumina" in name: + model_type = 'lumina' + elif "OmniGen" in name: + model_type = 'omnigen' + elif "CogVideo" in name: + model_type = 'cogvideox' + elif "Sana" in name: + model_type = 'sana' + else: + model_type = name + return model_type + + class ModelData: def __init__(self): self.sd_model = None @@ -82,36 +121,7 @@ def sd_model_type(self): if modules.sd_models.model_data.sd_model is None: model_type = 'none' return model_type - if not shared.native: - model_type = 'ldm' - elif "StableDiffusion3" in self.sd_model.__class__.__name__: - model_type = 'sd3' - elif "StableDiffusionXL" in self.sd_model.__class__.__name__: - model_type = 'sdxl' - elif "StableDiffusion" in self.sd_model.__class__.__name__: - model_type = 'sd' - elif "LatentConsistencyModel" in self.sd_model.__class__.__name__: - model_type = 'sd' # lcm is compatible with sd - elif "InstaFlowPipeline" in self.sd_model.__class__.__name__: - model_type = 'sd' # instaflow is compatible with sd - elif "AnimateDiffPipeline" in self.sd_model.__class__.__name__: - model_type = 'sd' # animatediff is compatible with sd - elif "Kandinsky" in self.sd_model.__class__.__name__: - model_type = 'kandinsky' - elif "HunyuanDiT" in self.sd_model.__class__.__name__: - model_type = 'hunyuandit' - elif "Cascade" in self.sd_model.__class__.__name__: - model_type = 'sc' - elif "AuraFlow" in self.sd_model.__class__.__name__: - model_type = 'auraflow' - elif "Flux" in self.sd_model.__class__.__name__: - model_type = 'f1' - elif "OmniGen" in self.sd_model.__class__.__name__: - model_type = 'omnigen' - elif "CogVideo" in self.sd_model.__class__.__name__: - model_type = 'cogvideox' - else: - model_type = self.sd_model.__class__.__name__ + model_type = get_model_type(self.sd_model) except Exception: model_type = 'unknown' return model_type @@ -123,18 +133,7 @@ def sd_refiner_type(self): if modules.sd_models.model_data.sd_refiner is None: model_type = 'none' return model_type - if not shared.native: - model_type = 'ldm' - elif "StableDiffusion3" in self.sd_refiner.__class__.__name__: - model_type = 'sd3' - elif "StableDiffusionXL" in self.sd_refiner.__class__.__name__: - model_type = 'sdxl' - elif "StableDiffusion" in self.sd_refiner.__class__.__name__: - model_type = 'sd' - elif "Kandinsky" in self.sd_refiner.__class__.__name__: - model_type = 'kandinsky' - else: - model_type = self.sd_refiner.__class__.__name__ + model_type = get_model_type(self.sd_refiner) except Exception: model_type = 'unknown' return model_type diff --git a/modules/pag/__init__.py b/modules/pag/__init__.py index 29cdee8ca..8fe54c198 100644 --- a/modules/pag/__init__.py +++ b/modules/pag/__init__.py @@ -20,7 +20,9 @@ def apply(p: processing.StableDiffusionProcessing): # pylint: disable=arguments- if sd_models.get_diffusers_task(shared.sd_model) != sd_models.DiffusersTaskType.TEXT_2_IMAGE: shared.log.warning(f'PAG: pipeline={c} not implemented') return None - if detect.is_sd15(c): + if 'PAG' in shared.sd_model.__class__.__name__: + pass + elif detect.is_sd15(c): orig_pipeline = shared.sd_model shared.sd_model = sd_models.switch_pipe(StableDiffusionPAGPipeline, shared.sd_model) elif detect.is_sdxl(c): @@ -32,13 +34,14 @@ def apply(p: processing.StableDiffusionProcessing): # pylint: disable=arguments- p.task_args['pag_scale'] = p.pag_scale p.task_args['pag_adaptive_scaling'] = p.pag_adaptive + p.task_args['pag_adaptive_scale'] = p.pag_adaptive pag_applied_layers = shared.opts.pag_apply_layers pag_applied_layers_index = pag_applied_layers.split() if len(pag_applied_layers) > 0 else [] pag_applied_layers_index = [p.strip() for p in pag_applied_layers_index] p.task_args['pag_applied_layers_index'] = pag_applied_layers_index if len(pag_applied_layers_index) > 0 else ['m0'] # Available layers: d[0-5], m[0], u[0-8] p.extra_generation_params["PAG scale"] = p.pag_scale p.extra_generation_params["PAG adaptive"] = p.pag_adaptive - shared.log.debug(f'{c}: args={p.task_args}') + # shared.log.debug(f'{c}: args={p.task_args}') def unapply(): diff --git a/modules/processing_vae.py b/modules/processing_vae.py index 77a89c512..772a48adc 100644 --- a/modules/processing_vae.py +++ b/modules/processing_vae.py @@ -114,10 +114,11 @@ def full_vae_decode(latents, model): else: # manual upcast and we restore it later model.vae.orig_dtype = model.vae.dtype model.vae = model.vae.to(dtype=torch.float32) - latents = latents.to(torch.float32) latents = latents.to(devices.device) if getattr(model.vae, "post_quant_conv", None) is not None: latents = latents.to(next(iter(model.vae.post_quant_conv.parameters())).dtype) + else: + latents = latents.to(model.vae.dtype) # normalize latents latents_mean = model.vae.config.get("latents_mean", None) diff --git a/modules/schedulers/scheduler_dpm_flowmatch.py b/modules/schedulers/scheduler_dpm_flowmatch.py index 1afe54498..69452aca9 100644 --- a/modules/schedulers/scheduler_dpm_flowmatch.py +++ b/modules/schedulers/scheduler_dpm_flowmatch.py @@ -22,7 +22,8 @@ def __init__(self, x, t0, t1, seed=None, **kwargs): t0, t1, self.sign = self.sort(t0, t1) w0 = kwargs.get("w0", torch.zeros_like(x)) if seed is None: - seed = torch.randint(0, 2**63 - 1, []).item() + seed = [torch.randint(0, 2**63 - 1, []).item()] + seed = [s.initial_seed() if isinstance(s, torch.Generator) else s for s in seed] self.batched = True try: assert len(seed) == x.shape[0] diff --git a/modules/sd_detect.py b/modules/sd_detect.py index 071a83d7e..1931b9077 100644 --- a/modules/sd_detect.py +++ b/modules/sd_detect.py @@ -71,6 +71,8 @@ def detect_pipeline(f: str, op: str = 'model', warning=True, quiet=False): guess = 'Stable Cascade' if 'pixart-sigma' in f.lower(): guess = 'PixArt-Sigma' + if 'sana' in f.lower(): + guess = 'Sana' if 'lumina-next' in f.lower(): guess = 'Lumina-Next' if 'kolors' in f.lower(): diff --git a/modules/sd_models.py b/modules/sd_models.py index 5d42e314b..661559ae9 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -705,6 +705,9 @@ def load_diffuser_force(model_type, checkpoint_info, diffusers_load_config, op=' elif model_type in ['PixArt-Sigma']: # forced pipeline from modules.model_pixart import load_pixart sd_model = load_pixart(checkpoint_info, diffusers_load_config) + elif model_type in ['Sana']: # forced pipeline + from modules.model_sana import load_sana + sd_model = load_sana(checkpoint_info, diffusers_load_config) elif model_type in ['Lumina-Next']: # forced pipeline from modules.model_lumina import load_lumina sd_model = load_lumina(checkpoint_info, diffusers_load_config) diff --git a/modules/sd_samplers.py b/modules/sd_samplers.py index d8416e5d9..1b1be2d1a 100644 --- a/modules/sd_samplers.py +++ b/modules/sd_samplers.py @@ -3,6 +3,7 @@ from modules import shared from modules.sd_samplers_common import samples_to_image_grid, sample_to_image # pylint: disable=unused-import + debug = shared.log.trace if os.environ.get('SD_SAMPLER_DEBUG', None) is not None else lambda *args, **kwargs: None debug('Trace: SAMPLER') all_samplers = [] @@ -75,15 +76,15 @@ def create_sampler(name, model): shared.log.debug(f'Sampler: sampler="{name}" config={config.options}') return sampler elif shared.native: - FlowModels = ['Flux', 'StableDiffusion3', 'Lumina', 'AuraFlow'] + FlowModels = ['Flux', 'StableDiffusion3', 'Lumina', 'AuraFlow', 'Sana'] if 'KDiffusion' in model.__class__.__name__: return None - if any(x in model.__class__.__name__ for x in FlowModels) and 'FlowMatch' not in name: - shared.log.warning(f'Sampler: default={current} target="{name}" class={model.__class__.__name__} linear scheduler unsupported') - return None if not any(x in model.__class__.__name__ for x in FlowModels) and 'FlowMatch' in name: shared.log.warning(f'Sampler: default={current} target="{name}" class={model.__class__.__name__} flow-match scheduler unsupported') return None + # if any(x in model.__class__.__name__ for x in FlowModels) and 'FlowMatch' not in name: + # shared.log.warning(f'Sampler: default={current} target="{name}" class={model.__class__.__name__} linear scheduler unsupported') + # return None sampler = config.constructor(model) if sampler is None: sampler = config.constructor(model) diff --git a/modules/sd_samplers_common.py b/modules/sd_samplers_common.py index 723f7b181..5297d0bcd 100644 --- a/modules/sd_samplers_common.py +++ b/modules/sd_samplers_common.py @@ -9,6 +9,7 @@ SamplerData = namedtuple('SamplerData', ['name', 'constructor', 'aliases', 'options']) approximation_indexes = { "Simple": 0, "Approximate": 1, "TAESD": 2, "Full VAE": 3 } +flow_models = ['f1', 'sd3', 'lumina', 'auraflow', 'sana'] warned = False queue_lock = threading.Lock() diff --git a/modules/sd_samplers_diffusers.py b/modules/sd_samplers_diffusers.py index 7c23d4342..c95ae0858 100644 --- a/modules/sd_samplers_diffusers.py +++ b/modules/sd_samplers_diffusers.py @@ -4,13 +4,12 @@ import inspect import diffusers from modules import shared, errors -from modules import sd_samplers_common +from modules.sd_samplers_common import SamplerData, flow_models debug = shared.log.trace if os.environ.get('SD_SAMPLER_DEBUG', None) is not None else lambda *args, **kwargs: None debug('Trace: SAMPLER') - try: from diffusers import ( CMStochasticIterativeScheduler, @@ -63,7 +62,7 @@ # prediction_type is ideally set in model as well, but it maybe needed that we do auto-detect of model type in the future 'All': { 'num_train_timesteps': 1000, 'beta_start': 0.0001, 'beta_end': 0.02, 'beta_schedule': 'linear', 'prediction_type': 'epsilon' }, - 'UniPC': { 'predict_x0': True, 'sample_max_value': 1.0, 'solver_order': 2, 'solver_type': 'bh2', 'thresholding': False, 'use_beta_sigmas': False, 'use_exponential_sigmas': False, 'use_karras_sigmas': False, 'lower_order_final': True, 'timestep_spacing': 'linspace', 'final_sigmas_type': 'zero', 'rescale_betas_zero_snr': False }, + 'UniPC': { 'predict_x0': True, 'sample_max_value': 1.0, 'solver_order': 2, 'solver_type': 'bh2', 'thresholding': False, 'use_beta_sigmas': False, 'use_exponential_sigmas': False, 'use_flow_sigmas': False, 'use_karras_sigmas': False, 'lower_order_final': True, 'timestep_spacing': 'linspace', 'final_sigmas_type': 'zero', 'rescale_betas_zero_snr': False }, 'DDIM': { 'clip_sample': False, 'set_alpha_to_one': True, 'steps_offset': 0, 'clip_sample_range': 1.0, 'sample_max_value': 1.0, 'timestep_spacing': 'leading', 'rescale_betas_zero_snr': False, 'thresholding': False }, 'Euler': { 'steps_offset': 0, 'interpolation_type': "linear", 'rescale_betas_zero_snr': False, 'final_sigmas_type': 'zero', 'timestep_spacing': 'linspace', 'use_beta_sigmas': False, 'use_exponential_sigmas': False, 'use_karras_sigmas': False }, @@ -72,11 +71,11 @@ 'Euler EDM': { 'sigma_schedule': "karras" }, 'Euler FlowMatch': { 'timestep_spacing': "linspace", 'shift': 1, 'use_dynamic_shifting': False, 'use_karras_sigmas': False, 'use_exponential_sigmas': False, 'use_beta_sigmas': False }, - 'DPM++': { 'solver_order': 2, 'thresholding': False, 'sample_max_value': 1.0, 'algorithm_type': "dpmsolver++", 'solver_type': "midpoint", 'lower_order_final': True, 'use_karras_sigmas': False, 'use_exponential_sigmas': False, 'use_beta_sigmas': False, 'final_sigmas_type': 'sigma_min' }, - 'DPM++ 1S': { 'thresholding': False, 'sample_max_value': 1.0, 'algorithm_type': "dpmsolver++", 'solver_type': "midpoint", 'lower_order_final': True, 'use_karras_sigmas': False, 'use_exponential_sigmas': False, 'use_beta_sigmas': False, 'use_lu_lambdas': False, 'final_sigmas_type': 'zero', 'timestep_spacing': 'linspace', 'solver_order': 1 }, - 'DPM++ 2M': { 'thresholding': False, 'sample_max_value': 1.0, 'algorithm_type': "dpmsolver++", 'solver_type': "midpoint", 'lower_order_final': True, 'use_karras_sigmas': False, 'use_exponential_sigmas': False, 'use_beta_sigmas': False, 'use_lu_lambdas': False, 'final_sigmas_type': 'zero', 'timestep_spacing': 'linspace', 'solver_order': 2 }, - 'DPM++ 3M': { 'thresholding': False, 'sample_max_value': 1.0, 'algorithm_type': "dpmsolver++", 'solver_type': "midpoint", 'lower_order_final': True, 'use_karras_sigmas': False, 'use_exponential_sigmas': False, 'use_beta_sigmas': False, 'use_lu_lambdas': False, 'final_sigmas_type': 'zero', 'timestep_spacing': 'linspace', 'solver_order': 3 }, - 'DPM++ 2M SDE': { 'thresholding': False, 'sample_max_value': 1.0, 'algorithm_type': "sde-dpmsolver++", 'solver_type': "midpoint", 'lower_order_final': True, 'use_karras_sigmas': False, 'use_exponential_sigmas': False, 'use_beta_sigmas': False, 'use_lu_lambdas': False, 'final_sigmas_type': 'zero', 'timestep_spacing': 'linspace', 'solver_order': 2 }, + 'DPM++': { 'solver_order': 2, 'thresholding': False, 'sample_max_value': 1.0, 'algorithm_type': "dpmsolver++", 'solver_type': "midpoint", 'lower_order_final': True, 'use_karras_sigmas': False, 'use_exponential_sigmas': False, 'use_flow_sigmas': False, 'use_beta_sigmas': False, 'final_sigmas_type': 'sigma_min' }, + 'DPM++ 1S': { 'thresholding': False, 'sample_max_value': 1.0, 'algorithm_type': "dpmsolver++", 'solver_type': "midpoint", 'lower_order_final': True, 'use_karras_sigmas': False, 'use_exponential_sigmas': False, 'use_flow_sigmas': False, 'use_beta_sigmas': False, 'use_lu_lambdas': False, 'final_sigmas_type': 'zero', 'timestep_spacing': 'linspace', 'solver_order': 1 }, + 'DPM++ 2M': { 'thresholding': False, 'sample_max_value': 1.0, 'algorithm_type': "dpmsolver++", 'solver_type': "midpoint", 'lower_order_final': True, 'use_karras_sigmas': False, 'use_exponential_sigmas': False, 'use_flow_sigmas': False, 'use_beta_sigmas': False, 'use_lu_lambdas': False, 'final_sigmas_type': 'zero', 'timestep_spacing': 'linspace', 'solver_order': 2 }, + 'DPM++ 3M': { 'thresholding': False, 'sample_max_value': 1.0, 'algorithm_type': "dpmsolver++", 'solver_type': "midpoint", 'lower_order_final': True, 'use_karras_sigmas': False, 'use_exponential_sigmas': False, 'use_flow_sigmas': False, 'use_beta_sigmas': False, 'use_lu_lambdas': False, 'final_sigmas_type': 'zero', 'timestep_spacing': 'linspace', 'solver_order': 3 }, + 'DPM++ 2M SDE': { 'thresholding': False, 'sample_max_value': 1.0, 'algorithm_type': "sde-dpmsolver++", 'solver_type': "midpoint", 'lower_order_final': True, 'use_karras_sigmas': False, 'use_exponential_sigmas': False, 'use_flow_sigmas': False, 'use_beta_sigmas': False, 'use_lu_lambdas': False, 'final_sigmas_type': 'zero', 'timestep_spacing': 'linspace', 'solver_order': 2 }, 'DPM++ 2M EDM': { 'solver_order': 2, 'solver_type': 'midpoint', 'final_sigmas_type': 'zero', 'algorithm_type': 'dpmsolver++' }, 'DPM++ Cosine': { 'solver_order': 2, 'sigma_schedule': "exponential", 'prediction_type': "v-prediction" }, 'DPM SDE': { 'use_karras_sigmas': False, 'use_exponential_sigmas': False, 'use_beta_sigmas': False, 'noise_sampler_seed': None, 'timestep_spacing': 'linspace', 'steps_offset': 0, }, @@ -92,8 +91,8 @@ 'Heun': { 'use_beta_sigmas': False, 'use_karras_sigmas': False, 'use_exponential_sigmas': False, 'timestep_spacing': 'linspace' }, 'Heun FlowMatch': { 'timestep_spacing': "linspace", 'shift': 1 }, - 'DEIS': { 'solver_order': 2, 'thresholding': False, 'sample_max_value': 1.0, 'algorithm_type': "deis", 'solver_type': "logrho", 'lower_order_final': True, 'timestep_spacing': 'linspace', 'use_karras_sigmas': False, 'use_exponential_sigmas': False, 'use_beta_sigmas': False }, - 'SA Solver': {'predictor_order': 2, 'corrector_order': 2, 'thresholding': False, 'lower_order_final': True, 'use_karras_sigmas': False, 'use_exponential_sigmas': False, 'use_beta_sigmas': False, 'timestep_spacing': 'linspace'}, + 'DEIS': { 'solver_order': 2, 'thresholding': False, 'sample_max_value': 1.0, 'algorithm_type': "deis", 'solver_type': "logrho", 'lower_order_final': True, 'timestep_spacing': 'linspace', 'use_karras_sigmas': False, 'use_exponential_sigmas': False, 'use_flow_sigmas': False, 'use_beta_sigmas': False }, + 'SA Solver': {'predictor_order': 2, 'corrector_order': 2, 'thresholding': False, 'lower_order_final': True, 'use_karras_sigmas': False, 'use_flow_sigmas': False, 'use_exponential_sigmas': False, 'use_beta_sigmas': False, 'timestep_spacing': 'linspace'}, 'DC Solver': { 'beta_start': 0.0001, 'beta_end': 0.02, 'solver_order': 2, 'prediction_type': "epsilon", 'thresholding': False, 'solver_type': 'bh2', 'lower_order_final': True, 'dc_order': 2, 'disable_corrector': [0] }, 'VDM Solver': { 'clip_sample_range': 2.0, }, 'LCM': { 'beta_start': 0.00085, 'beta_end': 0.012, 'beta_schedule': "scaled_linear", 'set_alpha_to_one': True, 'rescale_betas_zero_snr': False, 'thresholding': False, 'timestep_spacing': 'linspace' }, @@ -110,54 +109,54 @@ } samplers_data_diffusers = [ - sd_samplers_common.SamplerData('Default', None, [], {}), + SamplerData('Default', None, [], {}), - sd_samplers_common.SamplerData('UniPC', lambda model: DiffusionSampler('UniPC', UniPCMultistepScheduler, model), [], {}), - sd_samplers_common.SamplerData('DDIM', lambda model: DiffusionSampler('DDIM', DDIMScheduler, model), [], {}), - sd_samplers_common.SamplerData('Euler', lambda model: DiffusionSampler('Euler', EulerDiscreteScheduler, model), [], {}), - sd_samplers_common.SamplerData('Euler a', lambda model: DiffusionSampler('Euler a', EulerAncestralDiscreteScheduler, model), [], {}), - sd_samplers_common.SamplerData('Euler SGM', lambda model: DiffusionSampler('Euler SGM', EulerDiscreteScheduler, model), [], {}), - sd_samplers_common.SamplerData('Euler EDM', lambda model: DiffusionSampler('Euler EDM', EDMEulerScheduler, model), [], {}), - sd_samplers_common.SamplerData('Euler FlowMatch', lambda model: DiffusionSampler('Euler FlowMatch', FlowMatchEulerDiscreteScheduler, model), [], {}), + SamplerData('UniPC', lambda model: DiffusionSampler('UniPC', UniPCMultistepScheduler, model), [], {}), + SamplerData('DDIM', lambda model: DiffusionSampler('DDIM', DDIMScheduler, model), [], {}), + SamplerData('Euler', lambda model: DiffusionSampler('Euler', EulerDiscreteScheduler, model), [], {}), + SamplerData('Euler a', lambda model: DiffusionSampler('Euler a', EulerAncestralDiscreteScheduler, model), [], {}), + SamplerData('Euler SGM', lambda model: DiffusionSampler('Euler SGM', EulerDiscreteScheduler, model), [], {}), + SamplerData('Euler EDM', lambda model: DiffusionSampler('Euler EDM', EDMEulerScheduler, model), [], {}), + SamplerData('Euler FlowMatch', lambda model: DiffusionSampler('Euler FlowMatch', FlowMatchEulerDiscreteScheduler, model), [], {}), - sd_samplers_common.SamplerData('DPM++', lambda model: DiffusionSampler('DPM++', DPMSolverSinglestepScheduler, model), [], {}), - sd_samplers_common.SamplerData('DPM++ 1S', lambda model: DiffusionSampler('DPM++ 1S', DPMSolverMultistepScheduler, model), [], {}), - sd_samplers_common.SamplerData('DPM++ 2M', lambda model: DiffusionSampler('DPM++ 2M', DPMSolverMultistepScheduler, model), [], {}), - sd_samplers_common.SamplerData('DPM++ 3M', lambda model: DiffusionSampler('DPM++ 3M', DPMSolverMultistepScheduler, model), [], {}), - sd_samplers_common.SamplerData('DPM++ 2M SDE', lambda model: DiffusionSampler('DPM++ 2M SDE', DPMSolverMultistepScheduler, model), [], {}), - sd_samplers_common.SamplerData('DPM++ 2M EDM', lambda model: DiffusionSampler('DPM++ 2M EDM', EDMDPMSolverMultistepScheduler, model), [], {}), - sd_samplers_common.SamplerData('DPM++ Cosine', lambda model: DiffusionSampler('DPM++ 2M EDM', CosineDPMSolverMultistepScheduler, model), [], {}), - sd_samplers_common.SamplerData('DPM SDE', lambda model: DiffusionSampler('DPM SDE', DPMSolverSDEScheduler, model), [], {}), + SamplerData('DPM++', lambda model: DiffusionSampler('DPM++', DPMSolverSinglestepScheduler, model), [], {}), + SamplerData('DPM++ 1S', lambda model: DiffusionSampler('DPM++ 1S', DPMSolverMultistepScheduler, model), [], {}), + SamplerData('DPM++ 2M', lambda model: DiffusionSampler('DPM++ 2M', DPMSolverMultistepScheduler, model), [], {}), + SamplerData('DPM++ 3M', lambda model: DiffusionSampler('DPM++ 3M', DPMSolverMultistepScheduler, model), [], {}), + SamplerData('DPM++ 2M SDE', lambda model: DiffusionSampler('DPM++ 2M SDE', DPMSolverMultistepScheduler, model), [], {}), + SamplerData('DPM++ 2M EDM', lambda model: DiffusionSampler('DPM++ 2M EDM', EDMDPMSolverMultistepScheduler, model), [], {}), + SamplerData('DPM++ Cosine', lambda model: DiffusionSampler('DPM++ 2M EDM', CosineDPMSolverMultistepScheduler, model), [], {}), + SamplerData('DPM SDE', lambda model: DiffusionSampler('DPM SDE', DPMSolverSDEScheduler, model), [], {}), - sd_samplers_common.SamplerData('DPM2 FlowMatch', lambda model: DiffusionSampler('DPM2 FlowMatch', FlowMatchDPMSolverMultistepScheduler, model), [], {}), - sd_samplers_common.SamplerData('DPM2a FlowMatch', lambda model: DiffusionSampler('DPM2a FlowMatch', FlowMatchDPMSolverMultistepScheduler, model), [], {}), - sd_samplers_common.SamplerData('DPM2++ 2M FlowMatch', lambda model: DiffusionSampler('DPM2++ 2M FlowMatch', FlowMatchDPMSolverMultistepScheduler, model), [], {}), - sd_samplers_common.SamplerData('DPM2++ 2S FlowMatch', lambda model: DiffusionSampler('DPM2++ 2S FlowMatch', FlowMatchDPMSolverMultistepScheduler, model), [], {}), - sd_samplers_common.SamplerData('DPM2++ SDE FlowMatch', lambda model: DiffusionSampler('DPM2++ SDE FlowMatch', FlowMatchDPMSolverMultistepScheduler, model), [], {}), - sd_samplers_common.SamplerData('DPM2++ 2M SDE FlowMatch', lambda model: DiffusionSampler('DPM2++ 2M SDE FlowMatch', FlowMatchDPMSolverMultistepScheduler, model), [], {}), - sd_samplers_common.SamplerData('DPM2++ 3M SDE FlowMatch', lambda model: DiffusionSampler('DPM2++ 3M SDE FlowMatch', FlowMatchDPMSolverMultistepScheduler, model), [], {}), + SamplerData('DPM2 FlowMatch', lambda model: DiffusionSampler('DPM2 FlowMatch', FlowMatchDPMSolverMultistepScheduler, model), [], {}), + SamplerData('DPM2a FlowMatch', lambda model: DiffusionSampler('DPM2a FlowMatch', FlowMatchDPMSolverMultistepScheduler, model), [], {}), + SamplerData('DPM2++ 2M FlowMatch', lambda model: DiffusionSampler('DPM2++ 2M FlowMatch', FlowMatchDPMSolverMultistepScheduler, model), [], {}), + SamplerData('DPM2++ 2S FlowMatch', lambda model: DiffusionSampler('DPM2++ 2S FlowMatch', FlowMatchDPMSolverMultistepScheduler, model), [], {}), + SamplerData('DPM2++ SDE FlowMatch', lambda model: DiffusionSampler('DPM2++ SDE FlowMatch', FlowMatchDPMSolverMultistepScheduler, model), [], {}), + SamplerData('DPM2++ 2M SDE FlowMatch', lambda model: DiffusionSampler('DPM2++ 2M SDE FlowMatch', FlowMatchDPMSolverMultistepScheduler, model), [], {}), + SamplerData('DPM2++ 3M SDE FlowMatch', lambda model: DiffusionSampler('DPM2++ 3M SDE FlowMatch', FlowMatchDPMSolverMultistepScheduler, model), [], {}), - sd_samplers_common.SamplerData('Heun', lambda model: DiffusionSampler('Heun', HeunDiscreteScheduler, model), [], {}), - sd_samplers_common.SamplerData('Heun FlowMatch', lambda model: DiffusionSampler('Heun FlowMatch', FlowMatchHeunDiscreteScheduler, model), [], {}), + SamplerData('Heun', lambda model: DiffusionSampler('Heun', HeunDiscreteScheduler, model), [], {}), + SamplerData('Heun FlowMatch', lambda model: DiffusionSampler('Heun FlowMatch', FlowMatchHeunDiscreteScheduler, model), [], {}), - sd_samplers_common.SamplerData('DEIS', lambda model: DiffusionSampler('DEIS', DEISMultistepScheduler, model), [], {}), - sd_samplers_common.SamplerData('SA Solver', lambda model: DiffusionSampler('SA Solver', SASolverScheduler, model), [], {}), - sd_samplers_common.SamplerData('DC Solver', lambda model: DiffusionSampler('DC Solver', DCSolverMultistepScheduler, model), [], {}), - sd_samplers_common.SamplerData('VDM Solver', lambda model: DiffusionSampler('VDM Solver', VDMScheduler, model), [], {}), - sd_samplers_common.SamplerData('BDIA DDIM', lambda model: DiffusionSampler('BDIA DDIM g=0', BDIA_DDIMScheduler, model), [], {}), + SamplerData('DEIS', lambda model: DiffusionSampler('DEIS', DEISMultistepScheduler, model), [], {}), + SamplerData('SA Solver', lambda model: DiffusionSampler('SA Solver', SASolverScheduler, model), [], {}), + SamplerData('DC Solver', lambda model: DiffusionSampler('DC Solver', DCSolverMultistepScheduler, model), [], {}), + SamplerData('VDM Solver', lambda model: DiffusionSampler('VDM Solver', VDMScheduler, model), [], {}), + SamplerData('BDIA DDIM', lambda model: DiffusionSampler('BDIA DDIM g=0', BDIA_DDIMScheduler, model), [], {}), - sd_samplers_common.SamplerData('PNDM', lambda model: DiffusionSampler('PNDM', PNDMScheduler, model), [], {}), - sd_samplers_common.SamplerData('IPNDM', lambda model: DiffusionSampler('IPNDM', IPNDMScheduler, model), [], {}), - sd_samplers_common.SamplerData('DDPM', lambda model: DiffusionSampler('DDPM', DDPMScheduler, model), [], {}), - sd_samplers_common.SamplerData('LMSD', lambda model: DiffusionSampler('LMSD', LMSDiscreteScheduler, model), [], {}), - sd_samplers_common.SamplerData('KDPM2', lambda model: DiffusionSampler('KDPM2', KDPM2DiscreteScheduler, model), [], {}), - sd_samplers_common.SamplerData('KDPM2 a', lambda model: DiffusionSampler('KDPM2 a', KDPM2AncestralDiscreteScheduler, model), [], {}), - sd_samplers_common.SamplerData('CMSI', lambda model: DiffusionSampler('CMSI', CMStochasticIterativeScheduler, model), [], {}), + SamplerData('PNDM', lambda model: DiffusionSampler('PNDM', PNDMScheduler, model), [], {}), + SamplerData('IPNDM', lambda model: DiffusionSampler('IPNDM', IPNDMScheduler, model), [], {}), + SamplerData('DDPM', lambda model: DiffusionSampler('DDPM', DDPMScheduler, model), [], {}), + SamplerData('LMSD', lambda model: DiffusionSampler('LMSD', LMSDiscreteScheduler, model), [], {}), + SamplerData('KDPM2', lambda model: DiffusionSampler('KDPM2', KDPM2DiscreteScheduler, model), [], {}), + SamplerData('KDPM2 a', lambda model: DiffusionSampler('KDPM2 a', KDPM2AncestralDiscreteScheduler, model), [], {}), + SamplerData('CMSI', lambda model: DiffusionSampler('CMSI', CMStochasticIterativeScheduler, model), [], {}), - sd_samplers_common.SamplerData('LCM', lambda model: DiffusionSampler('LCM', LCMScheduler, model), [], {}), - sd_samplers_common.SamplerData('TCD', lambda model: DiffusionSampler('TCD', TCDScheduler, model), [], {}), + SamplerData('LCM', lambda model: DiffusionSampler('LCM', LCMScheduler, model), [], {}), + SamplerData('TCD', lambda model: DiffusionSampler('TCD', TCDScheduler, model), [], {}), - sd_samplers_common.SamplerData('Same as primary', None, [], {}), + SamplerData('Same as primary', None, [], {}), ] @@ -178,14 +177,14 @@ def __init__(self, name, constructor, model, **kwargs): orig_config = model.default_scheduler.scheduler_config else: orig_config = model.default_scheduler.config - for key, value in config.get(name, {}).items(): # apply diffusers per-scheduler defaults - self.config[key] = value debug(f'Sampler: diffusers="{self.config}"') debug(f'Sampler: original="{orig_config}"') for key, value in orig_config.items(): # apply model defaults if key in self.config: self.config[key] = value debug(f'Sampler: default="{self.config}"') + for key, value in config.get(name, {}).items(): # apply diffusers per-scheduler defaults + self.config[key] = value for key, value in kwargs.items(): # apply user args, if any if key in self.config: self.config[key] = value @@ -205,10 +204,14 @@ def __init__(self, name, constructor, model, **kwargs): if len(timesteps) == 0: if 'sigma_schedule' in self.config: self.config['sigma_schedule'] = shared.opts.schedulers_sigma if shared.opts.schedulers_sigma != 'default' else None - if shared.opts.schedulers_sigma == 'betas' and 'use_beta_sigmas' in self.config: + if shared.opts.schedulers_sigma == 'default' and shared.sd_model_type in flow_models and 'use_flow_sigmas' in self.config: + self.config['use_flow_sigmas'] = True + elif shared.opts.schedulers_sigma == 'betas' and 'use_beta_sigmas' in self.config: self.config['use_beta_sigmas'] = True elif shared.opts.schedulers_sigma == 'karras' and 'use_karras_sigmas' in self.config: self.config['use_karras_sigmas'] = True + elif shared.opts.schedulers_sigma == 'flowmatch' and 'use_flow_sigmas' in self.config: + self.config['use_flow_sigmas'] = True elif shared.opts.schedulers_sigma == 'exponential' and 'use_exponential_sigmas' in self.config: self.config['use_exponential_sigmas'] = True elif shared.opts.schedulers_sigma == 'lambdas' and 'use_lu_lambdas' in self.config: diff --git a/modules/shared_items.py b/modules/shared_items.py index 4d9b325c8..9abb64718 100644 --- a/modules/shared_items.py +++ b/modules/shared_items.py @@ -86,6 +86,14 @@ def get_pipelines(): 'Kolors': getattr(diffusers, 'KolorsPipeline', None), 'AuraFlow': getattr(diffusers, 'AuraFlowPipeline', None), 'CogView': getattr(diffusers, 'CogView3PlusPipeline', None), + 'Stable Cascade': getattr(diffusers, 'StableCascadeCombinedPipeline', None), + 'PixArt-Sigma': getattr(diffusers, 'PixArtSigmaPipeline', None), + 'HunyuanDiT': getattr(diffusers, 'HunyuanDiTPipeline', None), + 'Stable Diffusion 3': getattr(diffusers, 'StableDiffusion3Pipeline', None), + 'Stable Diffusion 3 Img2Img': getattr(diffusers, 'StableDiffusion3Img2ImgPipeline', None), + 'Lumina-Next': getattr(diffusers, 'LuminaText2ImgPipeline', None), + 'FLUX': getattr(diffusers, 'FluxPipeline', None), + 'Sana': getattr(diffusers, 'SanaPAGPipeline', None), } if hasattr(diffusers, 'OnnxStableDiffusionPipeline'): onnx_pipelines = { @@ -103,19 +111,10 @@ def get_pipelines(): pipelines.update(onnx_pipelines) # items that may rely on diffusers dev version - if hasattr(diffusers, 'StableCascadeCombinedPipeline'): - pipelines['Stable Cascade'] = getattr(diffusers, 'StableCascadeCombinedPipeline', None) - if hasattr(diffusers, 'PixArtSigmaPipeline'): - pipelines['PixArt-Sigma'] = getattr(diffusers, 'PixArtSigmaPipeline', None) - if hasattr(diffusers, 'HunyuanDiTPipeline'): - pipelines['HunyuanDiT'] = getattr(diffusers, 'HunyuanDiTPipeline', None) - if hasattr(diffusers, 'StableDiffusion3Pipeline'): - pipelines['Stable Diffusion 3'] = getattr(diffusers, 'StableDiffusion3Pipeline', None) - pipelines['Stable Diffusion 3 Img2Img'] = getattr(diffusers, 'StableDiffusion3Img2ImgPipeline', None) - if hasattr(diffusers, 'LuminaText2ImgPipeline'): - pipelines['Lumina-Next'] = getattr(diffusers, 'LuminaText2ImgPipeline', None) + """ if hasattr(diffusers, 'FluxPipeline'): pipelines['FLUX'] = getattr(diffusers, 'FluxPipeline', None) + """ for k, v in pipelines.items(): if k != 'Autodetect' and v is None: diff --git a/modules/ui_sections.py b/modules/ui_sections.py index f15edb4bd..fcf53cf70 100644 --- a/modules/ui_sections.py +++ b/modules/ui_sections.py @@ -276,11 +276,11 @@ def set_sampler_preset(preset): else: # shared.native with gr.Row(elem_classes=['flex-break']): - sampler_sigma = gr.Dropdown(label='Sigma method', elem_id=f"{tabname}_sampler_sigma", choices=['default', 'karras', 'betas', 'exponential', 'lambdas'], value=shared.opts.schedulers_sigma, type='value') + sampler_sigma = gr.Dropdown(label='Sigma method', elem_id=f"{tabname}_sampler_sigma", choices=['default', 'karras', 'betas', 'exponential', 'lambdas', 'flowmatch'], value=shared.opts.schedulers_sigma, type='value') sampler_spacing = gr.Dropdown(label='Timestep spacing', elem_id=f"{tabname}_sampler_spacing", choices=['default', 'linspace', 'leading', 'trailing'], value=shared.opts.schedulers_timestep_spacing, type='value') with gr.Row(elem_classes=['flex-break']): sampler_beta = gr.Dropdown(label='Beta schedule', elem_id=f"{tabname}_sampler_beta", choices=['default', 'linear', 'scaled', 'cosine'], value=shared.opts.schedulers_beta_schedule, type='value') - sampler_prediction = gr.Dropdown(label='Prediction method', elem_id=f"{tabname}_sampler_prediction", choices=['default', 'epsilon', 'sample', 'v_prediction'], value=shared.opts.schedulers_prediction_type, type='value') + sampler_prediction = gr.Dropdown(label='Prediction method', elem_id=f"{tabname}_sampler_prediction", choices=['default', 'epsilon', 'sample', 'v_prediction', 'flow_prediction'], value=shared.opts.schedulers_prediction_type, type='value') with gr.Row(elem_classes=['flex-break']): sampler_presets = gr.Dropdown(label='Timesteps presets', elem_id=f"{tabname}_sampler_presets", choices=['None', 'AYS SD15', 'AYS SDXL'], value='None', type='value') sampler_timesteps = gr.Textbox(label='Timesteps override', elem_id=f"{tabname}_sampler_timesteps", value=shared.opts.schedulers_timesteps) From 925ec08cf9cb881167a9de32ae131e50e2eb0d9a Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 16 Dec 2024 15:40:54 -0500 Subject: [PATCH 096/249] add ufogen Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 14 +- installer.py | 2 +- modules/schedulers/scheduler_ufogen.py | 528 +++++++++++++++++++++++++ modules/sd_samplers_diffusers.py | 3 + wiki | 2 +- 5 files changed, 543 insertions(+), 6 deletions(-) create mode 100644 modules/schedulers/scheduler_ufogen.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a07f154c..b174a7a37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,9 +3,14 @@ ## Update for 2024-12-16 - Sana: both 1.6B and 0.6B -- ControlNet: better Union results, support for ProMax and Tile +- ControlNet: + - better Union results + - support for new ProMax model + - support for Tile models - FreeScale: run optimized iterative generation of images at different scales -- Samplers: UniPC, DEIS, SA, DPM-Multistep: add FlowMatch sigma method and prediction type +- Samplers: + - UniPC, DEIS, SA, DPM-Multistep: add FlowMatch sigma method and prediction type + - UFOGen: new fast scheduler for use with distilled models and low step counts ### New models and integrations @@ -108,11 +113,12 @@ - **IPEX**: update to IPEX 2.5.10+xpu - **OpenVINO**: update to 2024.5.0 - **Sampler** improvements - - UniPC, DEIS, SA, DPM-Multistep: allow FlowMatch method + - UniPC, DEIS, SA, DPM-Multistep: allow FlowMatch sigma method and prediction type - Euler FlowMatch: add sigma methods (*karras/exponential/betas*) - Euler FlowMatch: allow using timestep presets to set sigmas - DPM FlowMatch: update all and add sigma methods - - BDIA-DDIM: *experimental* + - BDIA-DDIM: *experimental* new scheduler + - UFOGen: *experimental* new scheduler ### Fixes diff --git a/installer.py b/installer.py index a12b09d4d..18f2b9399 100644 --- a/installer.py +++ b/installer.py @@ -459,7 +459,7 @@ def check_python(supported_minors=[9, 10, 11, 12], reason=None): def check_diffusers(): if args.skip_all or args.skip_requirements: return - sha = '5fb3a985173efaae7ff381b9040c386751d643da' # diffusers commit hash + sha = '5ed761a6f2a6dad56031f4e3e32223bfbe2dda01' # diffusers commit hash pkg = pkg_resources.working_set.by_key.get('diffusers', None) minor = int(pkg.version.split('.')[1] if pkg is not None else 0) cur = opts.get('diffusers_version', '') if minor > 0 else '' diff --git a/modules/schedulers/scheduler_ufogen.py b/modules/schedulers/scheduler_ufogen.py new file mode 100644 index 000000000..e03dec3c0 --- /dev/null +++ b/modules/schedulers/scheduler_ufogen.py @@ -0,0 +1,528 @@ +# Copyright 2023 UC Berkeley Team and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DISCLAIMER: This file is strongly influenced by https://github.com/ermongroup/ddim and https://github.com/xuyanwu/SIDDMs-UFOGen + +import math +from dataclasses import dataclass +from typing import List, Optional, Tuple, Union + +import numpy as np +import torch + +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.utils import BaseOutput, logging +from diffusers.utils.torch_utils import randn_tensor +from diffusers.schedulers.scheduling_utils import SchedulerMixin + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +@dataclass +# Copied from diffusers.schedulers.scheduling_ddpm.DDPMSchedulerOutput with DDPM->UFOGen +class UFOGenSchedulerOutput(BaseOutput): + """ + Output class for the scheduler's `step` function output. + + Args: + prev_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + Computed sample `(x_{t-1})` of previous timestep. `prev_sample` should be used as next model input in the + denoising loop. + pred_original_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + The predicted denoised sample `(x_{0})` based on the model output from the current timestep. + `pred_original_sample` can be used to preview progress or for guidance. + """ + + prev_sample: torch.FloatTensor + pred_original_sample: Optional[torch.FloatTensor] = None + + +# Copied from diffusers.schedulers.scheduling_ddpm.betas_for_alpha_bar +def betas_for_alpha_bar( + num_diffusion_timesteps, + max_beta=0.999, + alpha_transform_type="cosine", +): + """ + Create a beta schedule that discretizes the given alpha_t_bar function, which defines the cumulative product of + (1-beta) over time from t = [0,1]. + + Contains a function alpha_bar that takes an argument t and transforms it to the cumulative product of (1-beta) up + to that part of the diffusion process. + + + Args: + num_diffusion_timesteps (`int`): the number of betas to produce. + max_beta (`float`): the maximum beta to use; use values lower than 1 to + prevent singularities. + alpha_transform_type (`str`, *optional*, default to `cosine`): the type of noise schedule for alpha_bar. + Choose from `cosine` or `exp` + + Returns: + betas (`np.ndarray`): the betas used by the scheduler to step the model outputs + """ + if alpha_transform_type == "cosine": + + def alpha_bar_fn(t): + return math.cos((t + 0.008) / 1.008 * math.pi / 2) ** 2 + + elif alpha_transform_type == "exp": + + def alpha_bar_fn(t): + return math.exp(t * -12.0) + + else: + raise ValueError(f"Unsupported alpha_tranform_type: {alpha_transform_type}") + + betas = [] + for i in range(num_diffusion_timesteps): + t1 = i / num_diffusion_timesteps + t2 = (i + 1) / num_diffusion_timesteps + betas.append(min(1 - alpha_bar_fn(t2) / alpha_bar_fn(t1), max_beta)) + return torch.tensor(betas, dtype=torch.float32) + + +# Copied from diffusers.schedulers.scheduling_ddim.rescale_zero_terminal_snr +def rescale_zero_terminal_snr(betas): + """ + Rescales betas to have zero terminal SNR Based on https://arxiv.org/pdf/2305.08891.pdf (Algorithm 1) + + + Args: + betas (`torch.FloatTensor`): + the betas that the scheduler is being initialized with. + + Returns: + `torch.FloatTensor`: rescaled betas with zero terminal SNR + """ + # Convert betas to alphas_bar_sqrt + alphas = 1.0 - betas + alphas_cumprod = torch.cumprod(alphas, dim=0) + alphas_bar_sqrt = alphas_cumprod.sqrt() + + # Store old values. + alphas_bar_sqrt_0 = alphas_bar_sqrt[0].clone() + alphas_bar_sqrt_T = alphas_bar_sqrt[-1].clone() + + # Shift so the last timestep is zero. + alphas_bar_sqrt -= alphas_bar_sqrt_T + + # Scale so the first timestep is back to the old value. + alphas_bar_sqrt *= alphas_bar_sqrt_0 / (alphas_bar_sqrt_0 - alphas_bar_sqrt_T) + + # Convert alphas_bar_sqrt to betas + alphas_bar = alphas_bar_sqrt**2 # Revert sqrt + alphas = alphas_bar[1:] / alphas_bar[:-1] # Revert cumprod + alphas = torch.cat([alphas_bar[0:1], alphas]) + betas = 1 - alphas + + return betas + + +class UFOGenScheduler(SchedulerMixin, ConfigMixin): + """ + `UFOGenScheduler` implements multistep and onestep sampling for a UFOGen model, introduced in + [UFOGen: You Forward Once Large Scale Text-to-Image Generation via Diffusion GANs](https://arxiv.org/abs/2311.09257) + by Yanwu Xu, Yang Zhao, Zhisheng Xiao, and Tingbo Hou. UFOGen is a varianet of the denoising diffusion GAN (DDGAN) + model designed for one-step sampling. + + This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. Check the superclass documentation for the generic + methods the library implements for all schedulers such as loading and saving. + + Args: + num_train_timesteps (`int`, defaults to 1000): + The number of diffusion steps to train the model. + beta_start (`float`, defaults to 0.0001): + The starting `beta` value of inference. + beta_end (`float`, defaults to 0.02): + The final `beta` value. + beta_schedule (`str`, defaults to `"linear"`): + The beta schedule, a mapping from a beta range to a sequence of betas for stepping the model. Choose from + `linear`, `scaled_linear`, or `squaredcos_cap_v2`. + clip_sample (`bool`, defaults to `True`): + Clip the predicted sample for numerical stability. + clip_sample_range (`float`, defaults to 1.0): + The maximum magnitude for sample clipping. Valid only when `clip_sample=True`. + set_alpha_to_one (`bool`, defaults to `True`): + Each diffusion step uses the alphas product value at that step and at the previous one. For the final step + there is no previous alpha. When this option is `True` the previous alpha product is fixed to `1`, + otherwise it uses the alpha value at step 0. + prediction_type (`str`, defaults to `epsilon`, *optional*): + Prediction type of the scheduler function; can be `epsilon` (predicts the noise of the diffusion process), + `sample` (directly predicts the noisy sample`) or `v_prediction` (see section 2.4 of [Imagen + Video](https://imagen.research.google/video/paper.pdf) paper). + thresholding (`bool`, defaults to `False`): + Whether to use the "dynamic thresholding" method. This is unsuitable for latent-space diffusion models such + as Stable Diffusion. + dynamic_thresholding_ratio (`float`, defaults to 0.995): + The ratio for the dynamic thresholding method. Valid only when `thresholding=True`. + sample_max_value (`float`, defaults to 1.0): + The threshold value for dynamic thresholding. Valid only when `thresholding=True`. + timestep_spacing (`str`, defaults to `"leading"`): + The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information. + steps_offset (`int`, defaults to 0): + An offset added to the inference steps. You can use a combination of `offset=1` and + `set_alpha_to_one=False` to make the last step use step 0 for the previous alpha product like in Stable + Diffusion. + rescale_betas_zero_snr (`bool`, defaults to `False`): + Whether to rescale the betas to have zero terminal SNR. This enables the model to generate very bright and + dark samples instead of limiting it to samples with medium brightness. Loosely related to + [`--offset_noise`](https://github.com/huggingface/diffusers/blob/74fd735eb073eb1d774b1ab4154a0876eb82f055/examples/dreambooth/train_dreambooth.py#L506). + denoising_step_size (`int`, defaults to 250): + The denoising step size parameter from the UFOGen paper. The number of steps used for training is roughly + `math.ceil(num_train_timesteps / denoising_step_size)`. + """ + + order = 1 + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + beta_start: float = 0.0001, + beta_end: float = 0.02, + beta_schedule: str = "linear", + trained_betas: Optional[Union[np.ndarray, List[float]]] = None, + clip_sample: bool = True, + set_alpha_to_one: bool = True, + prediction_type: str = "epsilon", + thresholding: bool = False, + dynamic_thresholding_ratio: float = 0.995, + clip_sample_range: float = 1.0, + sample_max_value: float = 1.0, + timestep_spacing: str = "leading", + steps_offset: int = 0, + rescale_betas_zero_snr: bool = False, + denoising_step_size: int = 250, + ): + if trained_betas is not None: + self.betas = torch.tensor(trained_betas, dtype=torch.float32) + elif beta_schedule == "linear": + self.betas = torch.linspace(beta_start, beta_end, num_train_timesteps, dtype=torch.float32) + elif beta_schedule == "scaled_linear": + # this schedule is very specific to the latent diffusion model. + self.betas = torch.linspace(beta_start**0.5, beta_end**0.5, num_train_timesteps, dtype=torch.float32) ** 2 + elif beta_schedule == "squaredcos_cap_v2": + # Glide cosine schedule + self.betas = betas_for_alpha_bar(num_train_timesteps) + elif beta_schedule == "sigmoid": + # GeoDiff sigmoid schedule + betas = torch.linspace(-6, 6, num_train_timesteps) + self.betas = torch.sigmoid(betas) * (beta_end - beta_start) + beta_start + else: + raise NotImplementedError(f"{beta_schedule} does is not implemented for {self.__class__}") + + # Rescale for zero SNR + if rescale_betas_zero_snr: + self.betas = rescale_zero_terminal_snr(self.betas) + + self.alphas = 1.0 - self.betas + self.alphas_cumprod = torch.cumprod(self.alphas, dim=0) + + # For the final step, there is no previous alphas_cumprod because we are already at 0 + # `set_alpha_to_one` decides whether we set this parameter simply to one or + # whether we use the final alpha of the "non-previous" one. + self.final_alpha_cumprod = torch.tensor(1.0) if set_alpha_to_one else self.alphas_cumprod[0] + + # standard deviation of the initial noise distribution + self.init_noise_sigma = 1.0 + + # setable values + self.custom_timesteps = False + self.num_inference_steps = None + self.timesteps = torch.from_numpy(np.arange(0, num_train_timesteps)[::-1].copy()) + + def scale_model_input(self, sample: torch.FloatTensor, timestep: Optional[int] = None) -> torch.FloatTensor: + """ + Ensures interchangeability with schedulers that need to scale the denoising model input depending on the + current timestep. + + Args: + sample (`torch.FloatTensor`): + The input sample. + timestep (`int`, *optional*): + The current timestep in the diffusion chain. + + Returns: + `torch.FloatTensor`: + A scaled input sample. + """ + return sample + + def set_timesteps( + self, + num_inference_steps: Optional[int] = None, + device: Union[str, torch.device] = None, + timesteps: Optional[List[int]] = None, + ): + """ + Sets the discrete timesteps used for the diffusion chain (to be run before inference). + + Args: + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, + `timesteps` must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to support arbitrary spacing between timesteps. If `None`, then the default + timestep spacing strategy of equal spacing between timesteps is used. If `timesteps` is passed, + `num_inference_steps` must be `None`. + + """ + if num_inference_steps is not None and timesteps is not None: + raise ValueError("Can only pass one of `num_inference_steps` or `custom_timesteps`.") + + if timesteps is not None: + for i in range(1, len(timesteps)): + if timesteps[i] >= timesteps[i - 1]: + raise ValueError("`custom_timesteps` must be in descending order.") + + if timesteps[0] >= self.config.num_train_timesteps: + raise ValueError( + f"`timesteps` must start before `self.config.train_timesteps`:" + f" {self.config.num_train_timesteps}." + ) + + timesteps = np.array(timesteps, dtype=np.int64) + self.custom_timesteps = True + else: + if num_inference_steps > self.config.num_train_timesteps: + raise ValueError( + f"`num_inference_steps`: {num_inference_steps} cannot be larger than `self.config.train_timesteps`:" + f" {self.config.num_train_timesteps} as the unet model trained with this scheduler can only handle" + f" maximal {self.config.num_train_timesteps} timesteps." + ) + + self.num_inference_steps = num_inference_steps + self.custom_timesteps = False + + # TODO: For now, handle special case when num_inference_steps == 1 separately + if num_inference_steps == 1: + # Set the timestep schedule to num_train_timesteps - 1 rather than 0 + # (that is, the one-step timestep schedule is always trailing rather than leading or linspace) + timesteps = np.array([self.config.num_train_timesteps - 1], dtype=np.int64) + else: + # TODO: For now, retain the DDPM timestep spacing logic + # "linspace", "leading", "trailing" corresponds to annotation of Table 2. of https://arxiv.org/abs/2305.08891 + if self.config.timestep_spacing == "linspace": + timesteps = ( + np.linspace(0, self.config.num_train_timesteps - 1, num_inference_steps) + .round()[::-1] + .copy() + .astype(np.int64) + ) + elif self.config.timestep_spacing == "leading": + step_ratio = self.config.num_train_timesteps // self.num_inference_steps + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + timesteps = (np.arange(0, num_inference_steps) * step_ratio).round()[::-1].copy().astype(np.int64) + timesteps += self.config.steps_offset + elif self.config.timestep_spacing == "trailing": + step_ratio = self.config.num_train_timesteps / self.num_inference_steps + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + timesteps = np.round(np.arange(self.config.num_train_timesteps, 0, -step_ratio)).astype(np.int64) + timesteps -= 1 + else: + raise ValueError( + f"{self.config.timestep_spacing} is not supported. Please make sure to choose one of 'linspace', 'leading' or 'trailing'." + ) + + self.timesteps = torch.from_numpy(timesteps).to(device) + + # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler._threshold_sample + def _threshold_sample(self, sample: torch.FloatTensor) -> torch.FloatTensor: + """ + "Dynamic thresholding: At each sampling step we set s to a certain percentile absolute pixel value in xt0 (the + prediction of x_0 at timestep t), and if s > 1, then we threshold xt0 to the range [-s, s] and then divide by + s. Dynamic thresholding pushes saturated pixels (those near -1 and 1) inwards, thereby actively preventing + pixels from saturation at each step. We find that dynamic thresholding results in significantly better + photorealism as well as better image-text alignment, especially when using very large guidance weights." + + https://arxiv.org/abs/2205.11487 + """ + dtype = sample.dtype + batch_size, channels, *remaining_dims = sample.shape + + if dtype not in (torch.float32, torch.float64): + sample = sample.float() # upcast for quantile calculation, and clamp not implemented for cpu half + + # Flatten sample for doing quantile calculation along each image + sample = sample.reshape(batch_size, channels * np.prod(remaining_dims)) + + abs_sample = sample.abs() # "a certain percentile absolute pixel value" + + s = torch.quantile(abs_sample, self.config.dynamic_thresholding_ratio, dim=1) + s = torch.clamp( + s, min=1, max=self.config.sample_max_value + ) # When clamped to min=1, equivalent to standard clipping to [-1, 1] + s = s.unsqueeze(1) # (batch_size, 1) because clamp will broadcast along dim=0 + sample = torch.clamp(sample, -s, s) / s # "we threshold xt0 to the range [-s, s] and then divide by s" + + sample = sample.reshape(batch_size, channels, *remaining_dims) + sample = sample.to(dtype) + + return sample + + def step( + self, + model_output: torch.FloatTensor, + timestep: int, + sample: torch.FloatTensor, + generator: Optional[torch.Generator] = None, + return_dict: bool = True, + ) -> Union[UFOGenSchedulerOutput, Tuple]: + """ + Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion + process from the learned model outputs (most often the predicted noise). + + Args: + model_output (`torch.FloatTensor`): + The direct output from learned diffusion model. + timestep (`float`): + The current discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + generator (`torch.Generator`, *optional*): + A random number generator. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~schedulers.scheduling_ufogen.UFOGenSchedulerOutput`] or `tuple`. + + Returns: + [`~schedulers.scheduling_ddpm.UFOGenSchedulerOutput`] or `tuple`: + If return_dict is `True`, [`~schedulers.scheduling_ufogen.UFOGenSchedulerOutput`] is returned, otherwise a + tuple is returned where the first element is the sample tensor. + + """ + # 0. Resolve timesteps + t = timestep + prev_t = self.previous_timestep(t) + + # 1. compute alphas, betas + alpha_prod_t = self.alphas_cumprod[t] + alpha_prod_t_prev = self.alphas_cumprod[prev_t] if prev_t >= 0 else self.final_alpha_cumprod + beta_prod_t = 1 - alpha_prod_t + # beta_prod_t_prev = 1 - alpha_prod_t_prev + # current_alpha_t = alpha_prod_t / alpha_prod_t_prev + # current_beta_t = 1 - current_alpha_t + + # 2. compute predicted original sample from predicted noise also called + # "predicted x_0" of formula (15) from https://arxiv.org/pdf/2006.11239.pdf + if self.config.prediction_type == "epsilon": + pred_original_sample = (sample - beta_prod_t ** (0.5) * model_output) / alpha_prod_t ** (0.5) + elif self.config.prediction_type == "sample": + pred_original_sample = model_output + elif self.config.prediction_type == "v_prediction": + pred_original_sample = (alpha_prod_t**0.5) * sample - (beta_prod_t**0.5) * model_output + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample` or" + " `v_prediction` for UFOGenScheduler." + ) + + # 3. Clip or threshold "predicted x_0" + if self.config.thresholding: + pred_original_sample = self._threshold_sample(pred_original_sample) + elif self.config.clip_sample: + pred_original_sample = pred_original_sample.clamp( + -self.config.clip_sample_range, self.config.clip_sample_range + ) + + # 4. Single-step or multi-step sampling + # Noise is not used on the final timestep of the timestep schedule. + # This also means that noise is not used for one-step sampling. + if t != self.timesteps[-1]: + # TODO: is this correct? + # Sample prev sample x_{t - 1} ~ q(x_{t - 1} | x_0 = G(x_t, t)) + device = model_output.device + noise = randn_tensor(model_output.shape, generator=generator, device=device, dtype=model_output.dtype) + sqrt_alpha_prod_t_prev = alpha_prod_t_prev**0.5 + sqrt_one_minus_alpha_prod_t_prev = (1 - alpha_prod_t_prev) ** 0.5 + pred_prev_sample = sqrt_alpha_prod_t_prev * pred_original_sample + sqrt_one_minus_alpha_prod_t_prev * noise + else: + # Simply return the pred_original_sample. If `prediction_type == "sample"`, this is equivalent to returning + # the output of the GAN generator U-Net on the initial noisy latents x_T ~ N(0, I). + pred_prev_sample = pred_original_sample + + if not return_dict: + return (pred_prev_sample,) + + return UFOGenSchedulerOutput(prev_sample=pred_prev_sample, pred_original_sample=pred_original_sample) + + # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler.add_noise + def add_noise( + self, + original_samples: torch.FloatTensor, + noise: torch.FloatTensor, + timesteps: torch.IntTensor, + ) -> torch.FloatTensor: + # Make sure alphas_cumprod and timestep have same device and dtype as original_samples + alphas_cumprod = self.alphas_cumprod.to(device=original_samples.device, dtype=original_samples.dtype) + timesteps = timesteps.to(original_samples.device) + + sqrt_alpha_prod = alphas_cumprod[timesteps] ** 0.5 + sqrt_alpha_prod = sqrt_alpha_prod.flatten() + while len(sqrt_alpha_prod.shape) < len(original_samples.shape): + sqrt_alpha_prod = sqrt_alpha_prod.unsqueeze(-1) + + sqrt_one_minus_alpha_prod = (1 - alphas_cumprod[timesteps]) ** 0.5 + sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.flatten() + while len(sqrt_one_minus_alpha_prod.shape) < len(original_samples.shape): + sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.unsqueeze(-1) + + noisy_samples = sqrt_alpha_prod * original_samples + sqrt_one_minus_alpha_prod * noise + return noisy_samples + + # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler.get_velocity + def get_velocity( + self, sample: torch.FloatTensor, noise: torch.FloatTensor, timesteps: torch.IntTensor + ) -> torch.FloatTensor: + # Make sure alphas_cumprod and timestep have same device and dtype as sample + alphas_cumprod = self.alphas_cumprod.to(device=sample.device, dtype=sample.dtype) + timesteps = timesteps.to(sample.device) + + sqrt_alpha_prod = alphas_cumprod[timesteps] ** 0.5 + sqrt_alpha_prod = sqrt_alpha_prod.flatten() + while len(sqrt_alpha_prod.shape) < len(sample.shape): + sqrt_alpha_prod = sqrt_alpha_prod.unsqueeze(-1) + + sqrt_one_minus_alpha_prod = (1 - alphas_cumprod[timesteps]) ** 0.5 + sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.flatten() + while len(sqrt_one_minus_alpha_prod.shape) < len(sample.shape): + sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.unsqueeze(-1) + + velocity = sqrt_alpha_prod * noise - sqrt_one_minus_alpha_prod * sample + return velocity + + def __len__(self): + return self.config.num_train_timesteps + + # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler.previous_timestep + def previous_timestep(self, timestep): + if self.custom_timesteps: + index = (self.timesteps == timestep).nonzero(as_tuple=True)[0][0] + if index == self.timesteps.shape[0] - 1: + prev_t = torch.tensor(-1) + else: + prev_t = self.timesteps[index + 1] + else: + num_inference_steps = ( + self.num_inference_steps if self.num_inference_steps else self.config.num_train_timesteps + ) + prev_t = timestep - self.config.num_train_timesteps // num_inference_steps + + return prev_t \ No newline at end of file diff --git a/modules/sd_samplers_diffusers.py b/modules/sd_samplers_diffusers.py index c95ae0858..6c05b2045 100644 --- a/modules/sd_samplers_diffusers.py +++ b/modules/sd_samplers_diffusers.py @@ -52,6 +52,7 @@ from modules.schedulers.scheduler_vdm import VDMScheduler # pylint: disable=ungrouped-imports from modules.schedulers.scheduler_dpm_flowmatch import FlowMatchDPMSolverMultistepScheduler # pylint: disable=ungrouped-imports from modules.schedulers.scheduler_bdia import BDIA_DDIMScheduler # pylint: disable=ungrouped-imports + from modules.schedulers.scheduler_ufogen import UFOGenScheduler # pylint: disable=ungrouped-imports except Exception as e: shared.log.error(f'Diffusers import error: version={diffusers.__version__} error: {e}') if os.environ.get('SD_SAMPLER_DEBUG', None) is not None: @@ -97,6 +98,7 @@ 'VDM Solver': { 'clip_sample_range': 2.0, }, 'LCM': { 'beta_start': 0.00085, 'beta_end': 0.012, 'beta_schedule': "scaled_linear", 'set_alpha_to_one': True, 'rescale_betas_zero_snr': False, 'thresholding': False, 'timestep_spacing': 'linspace' }, 'TCD': { 'set_alpha_to_one': True, 'rescale_betas_zero_snr': False, 'beta_schedule': 'scaled_linear' }, + 'UFOGen': {}, 'BDIA DDIM': { 'clip_sample': False, 'set_alpha_to_one': True, 'steps_offset': 0, 'clip_sample_range': 1.0, 'sample_max_value': 1.0, 'timestep_spacing': 'leading', 'rescale_betas_zero_snr': False, 'thresholding': False, 'gamma': 1.0 }, 'PNDM': { 'skip_prk_steps': False, 'set_alpha_to_one': False, 'steps_offset': 0, 'timestep_spacing': 'linspace' }, @@ -155,6 +157,7 @@ SamplerData('LCM', lambda model: DiffusionSampler('LCM', LCMScheduler, model), [], {}), SamplerData('TCD', lambda model: DiffusionSampler('TCD', TCDScheduler, model), [], {}), + SamplerData('UFOGen', lambda model: DiffusionSampler('UFOGen', UFOGenScheduler, model), [], {}), SamplerData('Same as primary', None, [], {}), ] diff --git a/wiki b/wiki index a4eaad83c..2870b888c 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit a4eaad83ccb8e82cb91fde4c038877616ed012d6 +Subproject commit 2870b888c1848930a93c8fd5475ffeb907f21384 From 4e4412064d7a793d91c1b1c3f31dc850fcdefb3d Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 16 Dec 2024 15:55:52 -0500 Subject: [PATCH 097/249] update sana Signed-off-by: Vladimir Mandic --- modules/model_sana.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/modules/model_sana.py b/modules/model_sana.py index 06e6fe981..eaecbceb8 100644 --- a/modules/model_sana.py +++ b/modules/model_sana.py @@ -11,12 +11,11 @@ def load_sana(checkpoint_info, diffusers_load_config={}): diffusers_load_config['variant'] = 'fp16' diffusers_load_config['torch_dtype'] = devices.dtype diffusers_load_config = model_quant.create_bnb_config(diffusers_load_config) - pipe = diffusers.SanaPAGPipeline.from_pretrained( + pipe = diffusers.SanaPipeline.from_pretrained( # SanaPAGPipeline repo_id, - # pag_applied_layers=["transformer_blocks.8"], cache_dir = shared.opts.diffusers_dir, **diffusers_load_config, - ).to(devices.dtype) + ) if shared.opts.diffusers_eval: pipe.text_encoder.eval() pipe.transformer.eval() From 19ed07599194b9057070da7671a5e03cd3a4a644 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 17 Dec 2024 09:05:06 -0500 Subject: [PATCH 098/249] update sana and changelog Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 15 +++--------- modules/model_sana.py | 53 +++++++++++++++++++++++++++++++++++-------- 2 files changed, 47 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b174a7a37..bfd14cacd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,24 +1,15 @@ # Change Log for SD.Next -## Update for 2024-12-16 - -- Sana: both 1.6B and 0.6B -- ControlNet: - - better Union results - - support for new ProMax model - - support for Tile models -- FreeScale: run optimized iterative generation of images at different scales -- Samplers: - - UniPC, DEIS, SA, DPM-Multistep: add FlowMatch sigma method and prediction type - - UFOGen: new fast scheduler for use with distilled models and low step counts +## Update for 2024-12-17 ### New models and integrations - [NVLabs Sana](https://huggingface.co/Efficient-Large-Model/Sana_1600M_1024px) **Sana** can synthesize high-resolution images with strong text-image alignment by using **Gemma2** as text-encoder + and its *fast* - typically at least **2x** faster than sd-xl even for 1.6B variant support for both 1.6B and 0.6B models to use, select from *networks -> models -> reference* and models will be auto-downloaded on first use - *reference values*: sampler: default, width/height: 1024, guidance scale: 4.5, attention guidance: 3.0, adaptive scaling: 0.0 + *reference values*: sampler: default (or any flow-match variant), width/height: 1024, guidance scale: 4.5 - [Flux Tools](https://blackforestlabs.ai/flux-1-tools/) **Redux** is actually a tool, **Fill** is inpaint/outpaint optimized version of *Flux-dev* **Canny** & **Depth** are optimized versions of *Flux-dev* for their respective tasks: they are *not* ControlNets that work on top of a model diff --git a/modules/model_sana.py b/modules/model_sana.py index eaecbceb8..f9986d726 100644 --- a/modules/model_sana.py +++ b/modules/model_sana.py @@ -1,24 +1,59 @@ +import time +import torch import diffusers -def load_sana(checkpoint_info, diffusers_load_config={}): +""" +Efficient-Large-Model/Sana_1600M_1024px_MultiLing_diffusers +Efficient-Large-Model/Sana_1600M_1024px_diffusers +Efficient-Large-Model/Sana_1600M_1024px_BF16_diffusers +Efficient-Large-Model/Sana_1600M_512px_MultiLing_diffusers +Efficient-Large-Model/Sana_1600M_512px_diffusers +Efficient-Large-Model/Sana_600M_1024px_diffusers +Efficient-Large-Model/Sana_600M_512px_diffusers +""" + + +def load_sana(checkpoint_info, kwargs={}): from modules import shared, sd_models, devices, modelloader, model_quant modelloader.hf_login() repo_id = checkpoint_info if isinstance(checkpoint_info, str) else checkpoint_info.path repo_id = sd_models.path_to_repo(repo_id) + kwargs.pop('load_connected_pipeline', None) + kwargs.pop('safety_checker', None) + kwargs.pop('requires_safety_checker', None) + kwargs.pop('torch_dtype', None) + + if 'Sana_1600M' in repo_id: + if devices.dtype == torch.bfloat16: + repo_id = 'Efficient-Large-Model/Sana_1600M_1024px_BF16_diffusers' + kwargs['variant'] = 'bf16' + kwargs['torch_dtype'] = devices.dtype + else: + repo_id = 'Efficient-Large-Model/Sana_1600M_1024px_diffusers' + kwargs['variant'] = 'fp16' + if 'Sana_600M' in repo_id: + repo_id = 'Efficient-Large-Model/Sana_600M_1024px_diffusers' + kwargs['variant'] = 'fp16' - diffusers_load_config['variant'] = 'fp16' - diffusers_load_config['torch_dtype'] = devices.dtype - diffusers_load_config = model_quant.create_bnb_config(diffusers_load_config) - pipe = diffusers.SanaPipeline.from_pretrained( # SanaPAGPipeline - repo_id, - cache_dir = shared.opts.diffusers_dir, - **diffusers_load_config, - ) + kwargs = model_quant.create_bnb_config(kwargs) + shared.log.debug(f'Load model: type=Sana repo="{repo_id}" args={kwargs}') + t0 = time.time() + pipe = diffusers.SanaPipeline.from_pretrained(repo_id, cache_dir = shared.opts.diffusers_dir, **kwargs) + if devices.dtype == torch.bfloat16 or devices.dtype == torch.float32: + pipe.transformer = pipe.transformer.to(dtype=devices.dtype) + pipe.text_encoder = pipe.text_encoder.to(dtype=devices.dtype) + pipe.vae = pipe.vae.to(dtype=devices.dtype) + if devices.dtype == torch.float16: + pipe.transformer = pipe.transformer.to(dtype=devices.dtype) + pipe.text_encoder = pipe.text_encoder.to(dtype=torch.float32) # gemma2 does not support fp16 + pipe.vae = pipe.vae.to(dtype=torch.float32) # dc-ae often overflows in fp16 if shared.opts.diffusers_eval: pipe.text_encoder.eval() pipe.transformer.eval() + t1 = time.time() + shared.log.debug(f'Load model: type=Sana target={devices.dtype} te={pipe.text_encoder.dtype} transformer={pipe.transformer.dtype} vae={pipe.vae.dtype} time={t1-t0:.2f}') devices.torch_gc() return pipe From f8311b8379b8d94a5f7d1db89790afe59653cf08 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 17 Dec 2024 09:38:40 -0500 Subject: [PATCH 099/249] update readme Signed-off-by: Vladimir Mandic --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 722041c93..e6eabeff6 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,7 @@ See [models overview](https://github.com/vladmandic/automatic/wiki/Models) for d - [StabilityAI Stable Video Diffusion](https://huggingface.co/stabilityai/stable-video-diffusion-img2vid) Base, XT 1.0, XT 1.1 - [StabilityAI Stable Cascade](https://github.com/Stability-AI/StableCascade) *Full* and *Lite* - [Black Forest Labs FLUX.1](https://blackforestlabs.ai/announcing-black-forest-labs/) Dev, Schnell +- [NVLabs Sana](https://nvlabs.github.io/Sana/) - [AuraFlow](https://huggingface.co/fal/AuraFlow) - [AlphaVLLM Lumina-Next-SFT](https://huggingface.co/Alpha-VLLM/Lumina-Next-SFT-diffusers) - [Playground AI](https://huggingface.co/playgroundai/playground-v2-256px-base) *v1, v2 256, v2 512, v2 1024 and latest v2.5* From adb00b245210843808e5579ab27c0df8de8d55cf Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 17 Dec 2024 13:29:36 -0500 Subject: [PATCH 100/249] add torchao Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 7 +++++++ TODO.md | 13 +++---------- installer.py | 5 ++++- modules/model_flux.py | 6 ++++-- modules/model_quant.py | 8 ++++---- modules/model_sana.py | 1 + modules/model_tools.py | 6 ++++-- modules/sd_models.py | 2 +- modules/sd_models_compile.py | 17 +++-------------- modules/shared.py | 24 ++++++++++++------------ 10 files changed, 43 insertions(+), 46 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bfd14cacd..a1b41e7a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -101,6 +101,13 @@ ### Updates +- **Quantization** + - Add `TorchAO` *pre* (during load) and *post* (during execution) quantization + **torchao** supports 4 different int-based and 3 float-based quantization schemes + This is in addition to existing support for: + - `BitsAndBytes` with 3 float-based quantization schemes + - `Optimium.Quanto` with 3 int-based and 2 float-based quantizations schemes + - `GGUF` with pre-quantized weights - **IPEX**: update to IPEX 2.5.10+xpu - **OpenVINO**: update to 2024.5.0 - **Sampler** improvements diff --git a/TODO.md b/TODO.md index 996da5ad9..fafbea8c3 100644 --- a/TODO.md +++ b/TODO.md @@ -10,21 +10,14 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma ## Future Candidates -- SD35 IPAdapter: -- SD35 LoRA: -- Flux IPAdapter: +- SD35 IPAdapter: +- Flux IPAdapter: - Flux NF4: -- SANA: - LTX-Video: -- TorchAO: +- GGUF: ## Other - IPAdapter negative: - Control API enhance scripts compatibility - PixelSmith: - -## Workaround in place - -- GGUF -- FlowMatch diff --git a/installer.py b/installer.py index 18f2b9399..2bb3df492 100644 --- a/installer.py +++ b/installer.py @@ -459,7 +459,7 @@ def check_python(supported_minors=[9, 10, 11, 12], reason=None): def check_diffusers(): if args.skip_all or args.skip_requirements: return - sha = '5ed761a6f2a6dad56031f4e3e32223bfbe2dda01' # diffusers commit hash + sha = '1524781b88ac1a082e755a030ba9d73cd6948e84' # diffusers commit hash pkg = pkg_resources.working_set.by_key.get('diffusers', None) minor = int(pkg.version.split('.')[1] if pkg is not None else 0) cur = opts.get('diffusers_version', '') if minor > 0 else '' @@ -483,6 +483,7 @@ def check_onnx(): def check_torchao(): + """ if args.skip_all or args.skip_requirements: return if installed('torchao', quiet=True): @@ -492,6 +493,8 @@ def check_torchao(): pip('uninstall --yes torchao', ignore=True, quiet=True, uv=False) for m in [m for m in sys.modules if m.startswith('torchao')]: del sys.modules[m] + """ + return def install_cuda(): diff --git a/modules/model_flux.py b/modules/model_flux.py index 8d6a02ef6..362f96d9b 100644 --- a/modules/model_flux.py +++ b/modules/model_flux.py @@ -197,6 +197,7 @@ def load_transformer(file_path): # triggered by opts.sd_unet change transformer = _transformer else: diffusers_load_config = model_quant.create_bnb_config(diffusers_load_config) + diffusers_load_config = model_quant.create_ao_config(diffusers_load_config) transformer = diffusers.FluxTransformer2DModel.from_single_file(file_path, **diffusers_load_config) if transformer is None: shared.log.error('Failed to load UNet model') @@ -322,8 +323,9 @@ def load_flux(checkpoint_info, diffusers_load_config): # triggered by opts.sd_ch shared.log.warning(f'Load model: type=FLUX component={c} dtype={kwargs[c].dtype} cast dtype={devices.dtype} recast') kwargs[c] = kwargs[c].to(dtype=devices.dtype) - allow_bnb = 'gguf' not in (sd_unet.loaded_unet or '') - kwargs = model_quant.create_bnb_config(kwargs, allow_bnb) + allow_quant = 'gguf' not in (sd_unet.loaded_unet or '') + kwargs = model_quant.create_bnb_config(kwargs, allow_quant) + kwargs = model_quant.create_ao_config(kwargs, allow_quant) if checkpoint_info.path.endswith('.safetensors') and os.path.isfile(checkpoint_info.path): pipe = diffusers.FluxPipeline.from_single_file(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **kwargs, **diffusers_load_config) else: diff --git a/modules/model_quant.py b/modules/model_quant.py index 5c0b40080..10e5eb99e 100644 --- a/modules/model_quant.py +++ b/modules/model_quant.py @@ -38,9 +38,9 @@ def create_ao_config(kwargs = None, allow_ao: bool = True): load_torchao() if ao is None: return kwargs - ao_config = {} - # ao_config = diffusers.TorchAoConfig("int8wo") # TODO torchao - shared.log.debug(f'Quantization: module=all type=bnb dtype={shared.opts.torchao_quantization_type}') + diffusers.utils.import_utils.is_torchao_available = lambda: True + ao_config = diffusers.TorchAoConfig(shared.opts.torchao_quantization_type) + shared.log.debug(f'Quantization: module=all type=torchao dtype={shared.opts.torchao_quantization_type}') if kwargs is None: return ao_config else: @@ -53,7 +53,7 @@ def load_torchao(msg='', silent=False): global ao # pylint: disable=global-statement if ao is not None: return ao - install('torchao', quiet=True) + install('torchao==0.7.0', quiet=True) try: import torchao ao = torchao diff --git a/modules/model_sana.py b/modules/model_sana.py index f9986d726..c25a7ffb9 100644 --- a/modules/model_sana.py +++ b/modules/model_sana.py @@ -38,6 +38,7 @@ def load_sana(checkpoint_info, kwargs={}): kwargs['variant'] = 'fp16' kwargs = model_quant.create_bnb_config(kwargs) + kwargs = model_quant.create_ao_config(kwargs) shared.log.debug(f'Load model: type=Sana repo="{repo_id}" args={kwargs}') t0 = time.time() pipe = diffusers.SanaPipeline.from_pretrained(repo_id, cache_dir = shared.opts.diffusers_dir, **kwargs) diff --git a/modules/model_tools.py b/modules/model_tools.py index 07cd61b6e..fdeda5c2a 100644 --- a/modules/model_tools.py +++ b/modules/model_tools.py @@ -69,11 +69,13 @@ def load_modules(repo_id: str, params: dict): subfolder = 'text_encoder_2' if cls == transformers.T5EncoderModel: # t5-xxl subfolder = 'text_encoder_3' - kwargs['quantization_config'] = model_quant.create_bnb_config() + kwargs = model_quant.create_bnb_config(kwargs) + kwargs = model_quant.create_ao_config(kwargs) kwargs['variant'] = 'fp16' if cls == diffusers.SD3Transformer2DModel: subfolder = 'transformer' - kwargs['quantization_config'] = model_quant.create_bnb_config() + kwargs = model_quant.create_bnb_config(kwargs) + kwargs = model_quant.create_ao_config(kwargs) if subfolder is None: continue shared.log.debug(f'Load: module={name} class={cls.__name__} repo={repo_id} location={subfolder}') diff --git a/modules/sd_models.py b/modules/sd_models.py index 661559ae9..52d0a1deb 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -282,7 +282,7 @@ def eval_model(model, op=None, sd_model=None): # pylint: disable=unused-argument model.eval() return model sd_model = sd_models_compile.apply_compile_to_model(sd_model, eval_model, ["Model", "VAE", "Text Encoder"], op="eval") - if len(shared.opts.torchao_quantization) > 0 and shared.opts.torchao_quantization_mode != 'post': + if len(shared.opts.torchao_quantization) > 0 and shared.opts.torchao_quantization_mode == 'post': sd_model = sd_models_compile.torchao_quantization(sd_model) if shared.opts.opt_channelslast and hasattr(sd_model, 'unet'): diff --git a/modules/sd_models_compile.py b/modules/sd_models_compile.py index 38d3ef57f..bdd47e2e1 100644 --- a/modules/sd_models_compile.py +++ b/modules/sd_models_compile.py @@ -505,24 +505,13 @@ def compile_diffusers(sd_model): def torchao_quantization(sd_model): try: - install('torchao', quiet=True) + install('torchao==0.7.0', quiet=True) from torchao import quantization as q except Exception as e: shared.log.error(f"Quantization: type=TorchAO quantization not supported: {e}") return sd_model - if shared.opts.torchao_quantization_type == "int8+act": - fn = q.int8_dynamic_activation_int8_weight - elif shared.opts.torchao_quantization_type == "int8": - fn = q.int8_weight_only - elif shared.opts.torchao_quantization_type == "int4": - fn = q.int4_weight_only - elif shared.opts.torchao_quantization_type == "fp8+act": - fn = q.float8_dynamic_activation_float8_weight - elif shared.opts.torchao_quantization_type == "fp8": - fn = q.float8_weight_only - elif shared.opts.torchao_quantization_type == "fpx": - fn = q.fpx_weight_only - else: + fn = getattr(q, shared.opts.torchao_quantization_type, None) + if fn is None: shared.log.error(f"Quantization: type=TorchAO type={shared.opts.torchao_quantization_type} not supported") return sd_model shared.log.info(f"Quantization: type=TorchAO pipe={sd_model.__class__.__name__} quant={shared.opts.torchao_quantization_type} fn={fn} targets={shared.opts.torchao_quantization}") diff --git a/modules/shared.py b/modules/shared.py index 23865ab63..b1196eba3 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -583,23 +583,23 @@ def get_default_modes(): options_templates.update(options_section(('quantization', "Quantization Settings"), { "bnb_sep": OptionInfo("

BitsAndBytes

", "", gr.HTML), - "bnb_quantization": OptionInfo([], "Enabled", gr.CheckboxGroup, {"choices": ["Model", "VAE", "Text Encoder"], "visible": native}), - "bnb_quantization_type": OptionInfo("nf4", "Type", gr.Radio, {"choices": ['nf4', 'fp8', 'fp4'], "visible": native}), - "bnb_quantization_storage": OptionInfo("uint8", "Backend storage", gr.Radio, {"choices": ["float16", "float32", "int8", "uint8", "float64", "bfloat16"], "visible": native}), + "bnb_quantization": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "VAE", "Text Encoder"], "visible": native}), + "bnb_quantization_type": OptionInfo("nf4", "Quantization type", gr.Dropdown, {"choices": ['nf4', 'fp8', 'fp4'], "visible": native}), + "bnb_quantization_storage": OptionInfo("uint8", "Backend storage", gr.Dropdown, {"choices": ["float16", "float32", "int8", "uint8", "float64", "bfloat16"], "visible": native}), "optimum_quanto_sep": OptionInfo("

Optimum Quanto

", "", gr.HTML), - "optimum_quanto_weights": OptionInfo([], "Enabled", gr.CheckboxGroup, {"choices": ["Model", "VAE", "Text Encoder", "ControlNet"], "visible": native}), - "optimum_quanto_weights_type": OptionInfo("qint8", "Type", gr.Radio, {"choices": ['qint8', 'qfloat8_e4m3fn', 'qfloat8_e5m2', 'qint4', 'qint2'], "visible": native}), - "optimum_quanto_activations_type": OptionInfo("none", "Activations ", gr.Radio, {"choices": ['none', 'qint8', 'qfloat8_e4m3fn', 'qfloat8_e5m2'], "visible": native}), + "optimum_quanto_weights": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "VAE", "Text Encoder", "ControlNet"], "visible": native}), + "optimum_quanto_weights_type": OptionInfo("qint8", "Quantization weights type", gr.Dropdown, {"choices": ['qint8', 'qfloat8_e4m3fn', 'qfloat8_e5m2', 'qint4', 'qint2'], "visible": native}), + "optimum_quanto_activations_type": OptionInfo("none", "Quantization activations type ", gr.Dropdown, {"choices": ['none', 'qint8', 'qfloat8_e4m3fn', 'qfloat8_e5m2'], "visible": native}), "torchao_sep": OptionInfo("

TorchAO

", "", gr.HTML), - "torchao_quantization": OptionInfo([], "Enabled", gr.CheckboxGroup, {"choices": ["Model", "VAE", "Text Encoder"], "visible": native}), - "torchao_quantization_mode": OptionInfo("pre", "Mode", gr.Radio, {"choices": ['pre', 'post'], "visible": native}), - "torchao_quantization_type": OptionInfo("int8", "Type", gr.Radio, {"choices": ["int8+act", "int8", "int4", "fp8+act", "fp8", "fpx"], "visible": native}), + "torchao_quantization": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "VAE", "Text Encoder"], "visible": native}), + "torchao_quantization_mode": OptionInfo("pre", "Quantization mode", gr.Dropdown, {"choices": ['pre', 'post'], "visible": native}), + "torchao_quantization_type": OptionInfo("int8_weight_only", "Quantization type", gr.Dropdown, {"choices": ['int4_weight_only', 'int8_dynamic_activation_int4_weight', 'int8_weight_only', 'int8_dynamic_activation_int8_weight', 'float8_weight_only', 'float8_dynamic_activation_float8_weight', 'float8_static_activation_float8_weight'], "visible": native}), "nncf_sep": OptionInfo("

NNCF

", "", gr.HTML), - "nncf_compress_weights": OptionInfo([], "Enabled", gr.CheckboxGroup, {"choices": ["Model", "VAE", "Text Encoder", "ControlNet"], "visible": native}), - "nncf_compress_weights_mode": OptionInfo("INT8", "Mode", gr.Radio, {"choices": ['INT8', 'INT8_SYM', 'INT4_ASYM', 'INT4_SYM', 'NF4'] if cmd_opts.use_openvino else ['INT8']}), + "nncf_compress_weights": OptionInfo([], "Quantization enabled", gr.CheckboxGroup, {"choices": ["Model", "VAE", "Text Encoder", "ControlNet"], "visible": native}), + "nncf_compress_weights_mode": OptionInfo("INT8", "Quantization type", gr.Dropdown, {"choices": ['INT8', 'INT8_SYM', 'INT4_ASYM', 'INT4_SYM', 'NF4'] if cmd_opts.use_openvino else ['INT8']}), "nncf_compress_weights_raito": OptionInfo(1.0, "Compress ratio", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01, "visible": cmd_opts.use_openvino}), "nncf_quantize": OptionInfo([], "OpenVINO enabled", gr.CheckboxGroup, {"choices": ["Model", "VAE", "Text Encoder"], "visible": cmd_opts.use_openvino}), - "nncf_quant_mode": OptionInfo("INT8", "OpenVINO mode", gr.Radio, {"choices": ['INT8', 'FP8_E4M3', 'FP8_E5M2'], "visible": cmd_opts.use_openvino}), + "nncf_quant_mode": OptionInfo("INT8", "OpenVINO mode", gr.Dropdown, {"choices": ['INT8', 'FP8_E4M3', 'FP8_E5M2'], "visible": cmd_opts.use_openvino}), "quant_shuffle_weights": OptionInfo(False, "Shuffle weights", gr.Checkbox, {"visible": native}), })) From d938e752d4516e2b29dfc4d013d14cf2a0c24e99 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Tue, 17 Dec 2024 22:24:25 +0300 Subject: [PATCH 101/249] Use apply_compile_model with torchao --- modules/sd_models_compile.py | 31 +++++++++---------------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/modules/sd_models_compile.py b/modules/sd_models_compile.py index bdd47e2e1..16540019c 100644 --- a/modules/sd_models_compile.py +++ b/modules/sd_models_compile.py @@ -47,7 +47,7 @@ def apply_compile_to_model(sd_model, function, options, op=None): sd_model.prior_pipe.prior.clip_txt_pooled_mapper = backup_clip_txt_pooled_mapper if "Text Encoder" in options: if hasattr(sd_model, 'text_encoder') and hasattr(sd_model.text_encoder, 'config'): - if hasattr(sd_model, 'decoder_pipe') and hasattr(sd_model.decoder_pipe, 'text_encoder'): + if hasattr(sd_model, 'decoder_pipe') and hasattr(sd_model.decoder_pipe, 'text_encoder') and hasattr(sd_model.decoder_pipe.text_encoder, 'config'): sd_model.decoder_pipe.text_encoder = function(sd_model.decoder_pipe.text_encoder, op="decoder_pipe.text_encoder", sd_model=sd_model) else: if op == "nncf" and sd_model.text_encoder.__class__.__name__ in {"T5EncoderModel", "UMT5EncoderModel"}: @@ -76,7 +76,7 @@ def apply_compile_to_model(sd_model, function, options, op=None): dtype=torch.float32 if devices.dtype != torch.bfloat16 else torch.bfloat16 ) sd_model.text_encoder_3 = function(sd_model.text_encoder_3, op="text_encoder_3", sd_model=sd_model) - if hasattr(sd_model, 'prior_pipe') and hasattr(sd_model.prior_pipe, 'text_encoder'): + if hasattr(sd_model, 'prior_pipe') and hasattr(sd_model.prior_pipe, 'text_encoder') and hasattr(sd_model.prior_pipe.text_encoder, 'config'): sd_model.prior_pipe.text_encoder = function(sd_model.prior_pipe.text_encoder, op="prior_pipe.text_encoder", sd_model=sd_model) if "VAE" in options: if hasattr(sd_model, 'vae') and hasattr(sd_model.vae, 'decode'): @@ -510,34 +510,21 @@ def torchao_quantization(sd_model): except Exception as e: shared.log.error(f"Quantization: type=TorchAO quantization not supported: {e}") return sd_model + fn = getattr(q, shared.opts.torchao_quantization_type, None) if fn is None: shared.log.error(f"Quantization: type=TorchAO type={shared.opts.torchao_quantization_type} not supported") return sd_model + def torchao_model(model, op=None, sd_model=None): + q.quantize_(model, fn(), device=devices.device) + return model + shared.log.info(f"Quantization: type=TorchAO pipe={sd_model.__class__.__name__} quant={shared.opts.torchao_quantization_type} fn={fn} targets={shared.opts.torchao_quantization}") try: t0 = time.time() - modules = [] - if hasattr(sd_model, 'unet') and 'Model' in shared.opts.torchao_quantization: - modules.append('unet') - q.quantize_(sd_model.unet, fn(), device=devices.device) - if hasattr(sd_model, 'transformer') and 'Model' in shared.opts.torchao_quantization: - modules.append('transformer') - q.quantize_(sd_model.transformer, fn(), device=devices.device) - if hasattr(sd_model, 'vae') and 'VAE' in shared.opts.torchao_quantization: - modules.append('vae') - q.quantize_(sd_model.vae, fn(), device=devices.device) - if hasattr(sd_model, 'text_encoder') and 'Text Encoder' in shared.opts.torchao_quantization: - modules.append('te1') - q.quantize_(sd_model.text_encoder, fn(), device=devices.device) - if hasattr(sd_model, 'text_encoder_2') and 'Text Encoder' in shared.opts.torchao_quantization: - modules.append('te2') - q.quantize_(sd_model.text_encoder_2, fn(), device=devices.device) - if hasattr(sd_model, 'text_encoder_3') and 'Text Encoder' in shared.opts.torchao_quantization: - modules.append('te3') - q.quantize_(sd_model.text_encoder_3, fn(), device=devices.device) + apply_compile_to_model(sd_model, torchao_model, shared.opts.torchao_quantization, op="torchao") t1 = time.time() - shared.log.info(f"Quantization: type=TorchAO modules={modules} time={t1-t0:.2f}") + shared.log.info(f"Quantization: type=TorchAO time={t1-t0:.2f}") except Exception as e: shared.log.error(f"Quantization: type=TorchAO {e}") setup_logging() # torchao uses dynamo which messes with logging so reset is needed From 9f8a404f9013cd24cecf573146db52f3feec1320 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 17 Dec 2024 14:35:17 -0500 Subject: [PATCH 102/249] update changelog Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 8 ++++---- README.md | 51 ++++----------------------------------------------- wiki | 2 +- 3 files changed, 9 insertions(+), 52 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a1b41e7a6..def6e8afe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,7 +31,7 @@ - [Flux ControlNet LoRA](https://huggingface.co/black-forest-labs/FLUX.1-Canny-dev-lora) alternative to standard ControlNets, FLUX.1 also allows LoRA to help guide the generation process both **Depth** and **Canny** LoRAs are available in standard control menus -- [StabilityAI SD35 ControlNets]([sd3_medium](https://huggingface.co/stabilityai/stable-diffusion-3.5-controlnets)) +- [StabilityAI SD35 ControlNets](https://huggingface.co/stabilityai/stable-diffusion-3.5-controlnets) - In addition to previously released `InstantX` and `Alimama`, we now have *official* ones from StabilityAI - [Style Aligned Image Generation](https://style-aligned-gen.github.io/) enable in scripts, compatible with sd-xl @@ -102,7 +102,7 @@ ### Updates - **Quantization** - - Add `TorchAO` *pre* (during load) and *post* (during execution) quantization + - Add `TorchAO` *pre* (during load) and *post* (during execution) quantization **torchao** supports 4 different int-based and 3 float-based quantization schemes This is in addition to existing support for: - `BitsAndBytes` with 3 float-based quantization schemes @@ -395,7 +395,7 @@ A month later and with nearly 300 commits, here is the latest [SD.Next](https:// #### New models for 2024-10-23 -- New fine-tuned [CLiP-ViT-L]((https://huggingface.co/zer0int/CLIP-GmP-ViT-L-14)) 1st stage **text-encoders** used by most models (SD15/SDXL/SD3/Flux/etc.) brings additional details to your images +- New fine-tuned [CLiP-ViT-L](https://huggingface.co/zer0int/CLIP-GmP-ViT-L-14) 1st stage **text-encoders** used by most models (SD15/SDXL/SD3/Flux/etc.) brings additional details to your images - New models: [Stable Diffusion 3.5 Large](https://huggingface.co/stabilityai/stable-diffusion-3.5-large) [OmniGen](https://arxiv.org/pdf/2409.11340) @@ -727,7 +727,7 @@ Examples: - vae is list of manually downloaded safetensors - text-encoder is list of predefined and manually downloaded text-encoders - **controlnet** support: - support for **InstantX/Shakker-Labs** models including [Union-Pro](InstantX/FLUX.1-dev-Controlnet-Union) + support for **InstantX/Shakker-Labs** models including [Union-Pro](https://huggingface.co/InstantX/FLUX.1-dev-Controlnet-Union) note that flux controlnet models are large, up to 6.6GB on top of already large base model! as such, you may need to use offloading:sequential which is not as fast, but uses far less memory when using union model, you must also select control mode in the control unit diff --git a/README.md b/README.md index e6eabeff6..53e76a319 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ - [Documentation](https://vladmandic.github.io/sdnext-docs/) - [SD.Next Features](#sdnext-features) -- [Model support](#model-support) +- [Model support](#model-support) and [Specifications]() - [Platform support](#platform-support) - [Getting started](#getting-started) @@ -35,7 +35,6 @@ All individual features are not listed here, instead check [ChangeLog](CHANGELOG - Platform specific autodetection and tuning performed on install - Optimized processing with latest `torch` developments with built-in support for `torch.compile` and multiple compile backends: *Triton, ZLUDA, StableFast, DeepCache, OpenVINO, NNCF, IPEX, OneDiff* -- Improved prompt parser - Built-in queue management - Enterprise level logging and hardened API - Built in installer with automatic updates and dependency management @@ -50,43 +49,13 @@ All individual features are not listed here, instead check [ChangeLog](CHANGELOG ![screenshot-modernui](https://github.com/user-attachments/assets/39e3bc9a-a9f7-4cda-ba33-7da8def08032) -For screenshots and informations on other available themes, see [Themes Wiki](https://github.com/vladmandic/automatic/wiki/Themes) +For screenshots and informations on other available themes, see [Themes Wiki](wiki/Themes.md)
## Model support -Additional models will be added as they become available and there is public interest in them -See [models overview](https://github.com/vladmandic/automatic/wiki/Models) for details on each model, including their architecture, complexity and other info - -- [RunwayML Stable Diffusion](https://github.com/Stability-AI/stablediffusion/) 1.x and 2.x *(all variants)* -- [StabilityAI Stable Diffusion XL](https://github.com/Stability-AI/generative-models), [StabilityAI Stable Diffusion 3.0](https://stability.ai/news/stable-diffusion-3-medium) Medium, [StabilityAI Stable Diffusion 3.5](https://huggingface.co/stabilityai/stable-diffusion-3.5-large) Medium, Large, Large Turbo -- [StabilityAI Stable Video Diffusion](https://huggingface.co/stabilityai/stable-video-diffusion-img2vid) Base, XT 1.0, XT 1.1 -- [StabilityAI Stable Cascade](https://github.com/Stability-AI/StableCascade) *Full* and *Lite* -- [Black Forest Labs FLUX.1](https://blackforestlabs.ai/announcing-black-forest-labs/) Dev, Schnell -- [NVLabs Sana](https://nvlabs.github.io/Sana/) -- [AuraFlow](https://huggingface.co/fal/AuraFlow) -- [AlphaVLLM Lumina-Next-SFT](https://huggingface.co/Alpha-VLLM/Lumina-Next-SFT-diffusers) -- [Playground AI](https://huggingface.co/playgroundai/playground-v2-256px-base) *v1, v2 256, v2 512, v2 1024 and latest v2.5* -- [Tencent HunyuanDiT](https://github.com/Tencent/HunyuanDiT) -- [OmniGen](https://arxiv.org/pdf/2409.11340) -- [Meissonic](https://github.com/viiika/Meissonic) -- [Kwai Kolors](https://huggingface.co/Kwai-Kolors/Kolors) -- [CogView 3+](https://huggingface.co/THUDM/CogView3-Plus-3B) -- [LCM: Latent Consistency Models](https://github.com/openai/consistency_models) -- [aMUSEd](https://huggingface.co/amused/amused-256) 256 and 512 -- [Segmind Vega](https://huggingface.co/segmind/Segmind-Vega), [Segmind SSD-1B](https://huggingface.co/segmind/SSD-1B), [Segmind SegMoE](https://github.com/segmind/segmoe) *SD and SD-XL*, [Segmind SD Distilled](https://huggingface.co/blog/sd_distillation) *(all variants)* -- [Kandinsky](https://github.com/ai-forever/Kandinsky-2) *2.1 and 2.2 and latest 3.0* -- [PixArt-α XL 2](https://github.com/PixArt-alpha/PixArt-alpha) *Medium and Large*, [PixArt-Σ](https://github.com/PixArt-alpha/PixArt-sigma) -- [Warp Wuerstchen](https://huggingface.co/blog/wuertschen) -- [Tsinghua UniDiffusion](https://github.com/thu-ml/unidiffuser) -- [DeepFloyd IF](https://github.com/deep-floyd/IF) *Medium and Large* -- [ModelScope T2V](https://huggingface.co/damo-vilab/text-to-video-ms-1.7b) -- [BLIP-Diffusion](https://dxli94.github.io/BLIP-Diffusion-website/) -- [KOALA 700M](https://github.com/youngwanLEE/sdxl-koala) -- [VGen](https://huggingface.co/ali-vilab/i2vgen-xl) -- [SDXS](https://github.com/IDKiro/sdxs) -- [Hyper-SD](https://huggingface.co/ByteDance/Hyper-SD) +SD.Next supports broad range of models: [supported models](wiki/Model-Support.md) and [model specs](wiki/Models.md) ## Platform support @@ -116,21 +85,9 @@ See [models overview](https://github.com/vladmandic/automatic/wiki/Models) for d > If you run into issues, check out [troubleshooting](https://github.com/vladmandic/automatic/wiki/Troubleshooting) and [debugging](https://github.com/vladmandic/automatic/wiki/Debug) guides > [!TIP] -> All command line options can also be set via env variable +> All command line options can also be set via env variable > For example `--debug` is same as `set SD_DEBUG=true` -## Backend support - -**SD.Next** supports two main backends: *Diffusers* and *Original*: - -- **Diffusers**: Based on new [Huggingface Diffusers](https://huggingface.co/docs/diffusers/index) implementation - Supports *all* models listed below - This backend is set as default for new installations -- **Original**: Based on [LDM](https://github.com/Stability-AI/stablediffusion) reference implementation and significantly expanded on by [A1111](https://github.com/AUTOMATIC1111/stable-diffusion-webui) - This backend and is fully compatible with most existing functionality and extensions written for *A1111 SDWebUI* - Supports **SD 1.x** and **SD 2.x** models - All other model types such as *SD-XL, LCM, Stable Cascade, PixArt, Playground, Segmind, Kandinsky, etc.* require backend **Diffusers** - ### Collab - We'd love to have additional maintainers (with comes with full repo rights). If you're interested, ping us! diff --git a/wiki b/wiki index 2870b888c..ab4707483 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 2870b888c1848930a93c8fd5475ffeb907f21384 +Subproject commit ab4707483b47ba661ad8c062cf48a19a0ca9abed From 1c060ca6ec30b38c3e19b852ae8d1a40b43e57cb Mon Sep 17 00:00:00 2001 From: Disty0 Date: Tue, 17 Dec 2024 22:43:30 +0300 Subject: [PATCH 103/249] IPEX enable empty_cache fix for WSL on PyTorch --- modules/intel/ipex/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/intel/ipex/__init__.py b/modules/intel/ipex/__init__.py index 147aa2798..94b0cd0d4 100644 --- a/modules/intel/ipex/__init__.py +++ b/modules/intel/ipex/__init__.py @@ -122,7 +122,7 @@ def ipex_init(): # pylint: disable=too-many-statements torch.cuda.traceback = torch.xpu.traceback # Memory: - if legacy and 'linux' in sys.platform and "WSL2" in os.popen("uname -a").read(): + if 'linux' in sys.platform and "WSL2" in os.popen("uname -a").read(): torch.xpu.empty_cache = lambda: None torch.cuda.empty_cache = torch.xpu.empty_cache From a3bdee5b0e009c701caf0c7da14e0e5b1b110145 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 17 Dec 2024 15:01:33 -0500 Subject: [PATCH 104/249] update readme Signed-off-by: Vladimir Mandic --- README.md | 24 +++++++++--------------- wiki | 2 +- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 53e76a319..e33b3112a 100644 --- a/README.md +++ b/README.md @@ -49,13 +49,13 @@ All individual features are not listed here, instead check [ChangeLog](CHANGELOG ![screenshot-modernui](https://github.com/user-attachments/assets/39e3bc9a-a9f7-4cda-ba33-7da8def08032) -For screenshots and informations on other available themes, see [Themes Wiki](wiki/Themes.md) +For screenshots and informations on other available themes, see [Themes](https://vladmandic.github.io/sdnext-docs/Themes/)
## Model support -SD.Next supports broad range of models: [supported models](wiki/Model-Support.md) and [model specs](wiki/Models.md) +SD.Next supports broad range of models: [supported models](https://vladmandic.github.io/sdnext-docs/Model-Support/) and [model specs](https://vladmandic.github.io/sdnext-docs/Models/) ## Platform support @@ -72,28 +72,22 @@ SD.Next supports broad range of models: [supported models](wiki/Model-Support.md ## Getting started -- Get started with **SD.Next** by following the [installation instructions](https://github.com/vladmandic/automatic/wiki/Installation) -- For more details, check out [advanced installation](https://github.com/vladmandic/automatic/wiki/Advanced-Install) guide -- List and explanation of [command line arguments](https://github.com/vladmandic/automatic/wiki/CLI-Arguments) +- Get started with **SD.Next** by following the [installation instructions](https://vladmandic.github.io/sdnext-docs/Installation/) +- For more details, check out [advanced installation](https://vladmandic.github.io/sdnext-docs/Advanced-Install/) guide +- List and explanation of [command line arguments](https://vladmandic.github.io/sdnext-docs/CLI-Arguments/) - Install walkthrough [video](https://www.youtube.com/watch?v=nWTnTyFTuAs) > [!TIP] > And for platform specific information, check out -> [WSL](https://github.com/vladmandic/automatic/wiki/WSL) | [Intel Arc](https://github.com/vladmandic/automatic/wiki/Intel-ARC) | [DirectML](https://github.com/vladmandic/automatic/wiki/DirectML) | [OpenVINO](https://github.com/vladmandic/automatic/wiki/OpenVINO) | [ONNX & Olive](https://github.com/vladmandic/automatic/wiki/ONNX-Runtime) | [ZLUDA](https://github.com/vladmandic/automatic/wiki/ZLUDA) | [AMD ROCm](https://github.com/vladmandic/automatic/wiki/AMD-ROCm) | [MacOS](https://github.com/vladmandic/automatic/wiki/MacOS-Python.md) | [nVidia](https://github.com/vladmandic/automatic/wiki/nVidia) +> [WSL](https://vladmandic.github.io/sdnext-docs/WSL/) | [Intel Arc](https://vladmandic.github.io/sdnext-docs/Intel-ARC/) | [DirectML](https://vladmandic.github.io/sdnext-docs/DirectML/) | [OpenVINO](https://vladmandic.github.io/sdnext-docs/OpenVINO/) | [ONNX & Olive](https://vladmandic.github.io/sdnext-docs/ONNX-Runtime/) | [ZLUDA](https://vladmandic.github.io/sdnext-docs/ZLUDA/) | [AMD ROCm](https://vladmandic.github.io/sdnext-docs/AMD-ROCm/) | [MacOS](https://vladmandic.github.io/sdnext-docs/MacOS-Python/) | [nVidia](https://vladmandic.github.io/sdnext-docs/nVidia/) | [Docker](https://vladmandic.github.io/sdnext-docs/Docker/) > [!WARNING] -> If you run into issues, check out [troubleshooting](https://github.com/vladmandic/automatic/wiki/Troubleshooting) and [debugging](https://github.com/vladmandic/automatic/wiki/Debug) guides +> If you run into issues, check out [troubleshooting](https://vladmandic.github.io/sdnext-docs/Troubleshooting/) and [debugging](https://vladmandic.github.io/sdnext-docs/Debug/) guides > [!TIP] > All command line options can also be set via env variable > For example `--debug` is same as `set SD_DEBUG=true` -### Collab - -- We'd love to have additional maintainers (with comes with full repo rights). If you're interested, ping us! -- In addition to general cross-platform code, desire is to have a lead for each of the main platforms -This should be fully cross-platform, but we'd really love to have additional contributors and/or maintainers to join and help lead the efforts on different platforms - ### Credits - Main credit goes to [Automatic1111 WebUI](https://github.com/AUTOMATIC1111/stable-diffusion-webui) for the original codebase @@ -113,8 +107,8 @@ This should be fully cross-platform, but we'd really love to have additional con ### Docs -If you're unsure how to use a feature, best place to start is [Docs](https://vladmandic.github.io/sdnext-docs/) or [Wiki](https://github.com/vladmandic/automatic/wiki) and if its not there, -check [ChangeLog](CHANGELOG.md) for when feature was first introduced as it will always have a short note on how to use it +If you're unsure how to use a feature, best place to start is [Docs](https://vladmandic.github.io/sdnext-docs/) and if its not there, +check [ChangeLog](https://vladmandic.github.io/sdnext-docs/CHANGELOG/) for when feature was first introduced as it will always have a short note on how to use it ### Sponsors diff --git a/wiki b/wiki index ab4707483..dfaa5e491 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit ab4707483b47ba661ad8c062cf48a19a0ca9abed +Subproject commit dfaa5e4919604fd8e77a41756585a3a6fcb1b0bc From db08fbd207b535450ae160d5c4dd1dcf8c2ea7d2 Mon Sep 17 00:00:00 2001 From: Seunghoon Lee Date: Wed, 18 Dec 2024 20:51:25 +0900 Subject: [PATCH 105/249] relpath fails when venv is in different drive --- installer.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/installer.py b/installer.py index 2bb3df492..c0d3c907d 100644 --- a/installer.py +++ b/installer.py @@ -1177,9 +1177,15 @@ def same(ver): def check_venv(): + def try_relpath(p): + try: + return os.path.relpath(p) + except ValueError: + return p + import site - pkg_path = [os.path.relpath(p) for p in site.getsitepackages() if os.path.exists(p)] - log.debug(f'Packages: venv={os.path.relpath(sys.prefix)} site={pkg_path}') + pkg_path = [try_relpath(p) for p in site.getsitepackages() if os.path.exists(p)] + log.debug(f'Packages: venv={try_relpath(sys.prefix)} site={pkg_path}') for p in pkg_path: invalid = [] for f in os.listdir(p): From c03464418d80955c5dd0ebb739bf6e3ae2f0aa9d Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 18 Dec 2024 08:29:12 -0500 Subject: [PATCH 106/249] update wiki Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 2 +- modules/images_resize.py | 2 +- wiki | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index def6e8afe..6978c4e57 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2242,7 +2242,7 @@ Also new is support for **SDXL-Turbo** as well as new **Kandinsky 3** models and - in *Advanced* params - allows control of *latent clamping*, *color centering* and *range maximization* - supported by *XYZ grid* - - [SD21 Turbo](https://huggingface.co/stabilityai/sd-turbo) and [SDXL Turbo]() support + - [SD21 Turbo](https://huggingface.co/stabilityai/sd-turbo) and [SDXL Turbo](https://huggingface.co/stabilityai/sdxl-turbo) support - just set CFG scale (0.0-1.0) and steps (1-3) to a very low value - compatible with original StabilityAI SDXL-Turbo or any of the newer merges - download safetensors or select from networks -> reference diff --git a/modules/images_resize.py b/modules/images_resize.py index 5cf3e57e4..5b7c816f8 100644 --- a/modules/images_resize.py +++ b/modules/images_resize.py @@ -122,7 +122,7 @@ def context_aware(im, width, height, context): from modules import masking res = fill(im, color=0) res, _mask = masking.outpaint(res) - elif resize_mode == 5: # context-aware + elif resize_mode == 5: # context-aware res = context_aware(im, width, height, context) else: res = im.copy() diff --git a/wiki b/wiki index dfaa5e491..4dc357d28 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit dfaa5e4919604fd8e77a41756585a3a6fcb1b0bc +Subproject commit 4dc357d28956b827eb38df626e834fb6a25ec47c From dfe695b1bb41cc49291e29dd510c449cabf089a0 Mon Sep 17 00:00:00 2001 From: Seunghoon Lee Date: Wed, 18 Dec 2024 23:04:43 +0900 Subject: [PATCH 107/249] zluda HIP SDK 6.2 support --- modules/zluda_installer.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/zluda_installer.py b/modules/zluda_installer.py index 84c130e8d..5e43a6635 100644 --- a/modules/zluda_installer.py +++ b/modules/zluda_installer.py @@ -31,7 +31,10 @@ def install(zluda_path: os.PathLike) -> None: if os.path.exists(zluda_path): return - urllib.request.urlretrieve(f'https://github.com/lshqqytiger/ZLUDA/releases/download/rel.{os.environ.get("ZLUDA_HASH", "c0804ca624963aab420cb418412b1c7fbae3454b")}/ZLUDA-windows-rocm{rocm.version[0]}-amd64.zip', '_zluda') + commit = os.environ.get("ZLUDA_HASH", "1b6e012d8f2404840b524e2abae12cb91e1ac01d") + if rocm.version == "6.1": + commit = "c0804ca624963aab420cb418412b1c7fbae3454b" + urllib.request.urlretrieve(f'https://github.com/lshqqytiger/ZLUDA/releases/download/rel.{commit}/ZLUDA-windows-rocm{rocm.version[0]}-amd64.zip', '_zluda') with zipfile.ZipFile('_zluda', 'r') as archive: infos = archive.infolist() for info in infos: From 2ba0193c5a76ee9d4371aecdb074a4ca18abace6 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 18 Dec 2024 09:16:13 -0500 Subject: [PATCH 108/249] add sana chi Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 9 ++++++--- installer.py | 2 +- modules/processing_args.py | 5 +++++ wiki | 2 +- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6978c4e57..3211a4752 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,15 +1,18 @@ # Change Log for SD.Next -## Update for 2024-12-17 +## Update for 2024-12-18 ### New models and integrations - [NVLabs Sana](https://huggingface.co/Efficient-Large-Model/Sana_1600M_1024px) - **Sana** can synthesize high-resolution images with strong text-image alignment by using **Gemma2** as text-encoder - and its *fast* - typically at least **2x** faster than sd-xl even for 1.6B variant support for both 1.6B and 0.6B models + **Sana** can synthesize high-resolution images with strong text-image alignment by using **Gemma2** as text-encoder + and its *fast* - typically at least **2x** faster than sd-xl even for 1.6B variant and maintains performance regardless of resolution + e.g., rendering at 4k is possible in less than 8GB vram to use, select from *networks -> models -> reference* and models will be auto-downloaded on first use *reference values*: sampler: default (or any flow-match variant), width/height: 1024, guidance scale: 4.5 + *note* like other LLM-based text-encoders, sana prefers long and descriptive prompts + any short prompt below 300 characters will be auto-expanded using built in Gemma LLM before encoding while long prompts will be passed as-is - [Flux Tools](https://blackforestlabs.ai/flux-1-tools/) **Redux** is actually a tool, **Fill** is inpaint/outpaint optimized version of *Flux-dev* **Canny** & **Depth** are optimized versions of *Flux-dev* for their respective tasks: they are *not* ControlNets that work on top of a model diff --git a/installer.py b/installer.py index c0d3c907d..3dd0f13d5 100644 --- a/installer.py +++ b/installer.py @@ -459,7 +459,7 @@ def check_python(supported_minors=[9, 10, 11, 12], reason=None): def check_diffusers(): if args.skip_all or args.skip_requirements: return - sha = '1524781b88ac1a082e755a030ba9d73cd6948e84' # diffusers commit hash + sha = '862a7d5038c1c53641ffcab146a7eeb5ab683656' # diffusers commit hash pkg = pkg_resources.working_set.by_key.get('diffusers', None) minor = int(pkg.version.split('.')[1] if pkg is not None else 0) cur = opts.get('diffusers_version', '') if minor > 0 else '' diff --git a/modules/processing_args.py b/modules/processing_args.py index e7f53ba8e..d0afb6722 100644 --- a/modules/processing_args.py +++ b/modules/processing_args.py @@ -164,6 +164,11 @@ def set_pipeline_args(p, model, prompts: list, negative_prompts: list, prompts_2 args['negative_prompt'] = negative_prompts[0] else: args['negative_prompt'] = negative_prompts + if 'complex_human_instruction' in possible: + chi = any(len(p) < 300 for p in prompts) + p.extra_generation_params["CHI"] = chi + if not chi: + args['complex_human_instruction'] = None if prompt_parser_diffusers.embedder is not None and not prompt_parser_diffusers.embedder.scheduled_prompt: # not scheduled so we dont need it anymore prompt_parser_diffusers.embedder = None diff --git a/wiki b/wiki index 4dc357d28..470e75f0c 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 4dc357d28956b827eb38df626e834fb6a25ec47c +Subproject commit 470e75f0c70a22ed3d65187c70f04c131400b35d From 176acb5f510c70e96887d200aeb30c16ed1c86ce Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 18 Dec 2024 13:02:22 -0500 Subject: [PATCH 109/249] profiling Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 1 + modules/errors.py | 5 +++-- modules/ipadapter.py | 7 ++++++- modules/processing.py | 4 +++- modules/processing_diffusers.py | 4 ++++ modules/prompt_parser_diffusers.py | 2 ++ modules/shared.py | 5 ++++- modules/textual_inversion/textual_inversion.py | 2 ++ modules/timer.py | 6 +++++- 9 files changed, 30 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3211a4752..1931bd2b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -124,6 +124,7 @@ ### Fixes - add `SD_NO_CACHE=true` env variable to disable file/folder caching +- add settings -> networks -> embeddings -> enable/disable - update `diffusers` - fix README links - fix sdxl controlnet single-file loader diff --git a/modules/errors.py b/modules/errors.py index 527884cf1..5f2c54cb7 100644 --- a/modules/errors.py +++ b/modules/errors.py @@ -59,14 +59,14 @@ def exception(suppress=[]): console.print_exception(show_locals=False, max_frames=16, extra_lines=2, suppress=suppress, theme="ansi_dark", word_wrap=False, width=min([console.width, 200])) -def profile(profiler, msg: str, n: int = 5): +def profile(profiler, msg: str, n: int = 16): profiler.disable() import io import pstats stream = io.StringIO() # pylint: disable=abstract-class-instantiated p = pstats.Stats(profiler, stream=stream) p.sort_stats(pstats.SortKey.CUMULATIVE) - p.print_stats(100) + p.print_stats(200) # p.print_title() # p.print_call_heading(10, 'time') # p.print_callees(10) @@ -81,6 +81,7 @@ def profile(profiler, msg: str, n: int = 5): and '_lsprof' not in x and '/profiler' not in x and 'rich' not in x + and 'profile_torch' not in x and x.strip() != '' ] txt = '\n'.join(lines[:min(n, len(lines))]) diff --git a/modules/ipadapter.py b/modules/ipadapter.py index d5bfbec8c..4e93a6eee 100644 --- a/modules/ipadapter.py +++ b/modules/ipadapter.py @@ -14,6 +14,7 @@ clip_repo = "h94/IP-Adapter" clip_loaded = None +adapters_loaded = [] ADAPTERS_NONE = { 'None': { 'name': 'none', 'repo': 'none', 'subfolder': 'none' }, } @@ -129,9 +130,12 @@ def crop_images(images, crops): def unapply(pipe): # pylint: disable=arguments-differ + if len(adapters_loaded) == 0: + return try: if hasattr(pipe, 'set_ip_adapter_scale'): pipe.set_ip_adapter_scale(0) + pipe.unload_ip_adapter() if hasattr(pipe, 'unet') and hasattr(pipe.unet, 'config') and pipe.unet.config.encoder_hid_dim_type == 'ip_image_proj': pipe.unet.encoder_hid_proj = None pipe.config.encoder_hid_dim_type = None @@ -141,7 +145,7 @@ def unapply(pipe): # pylint: disable=arguments-differ def apply(pipe, p: processing.StableDiffusionProcessing, adapter_names=[], adapter_scales=[1.0], adapter_crops=[False], adapter_starts=[0.0], adapter_ends=[1.0], adapter_images=[]): - global clip_loaded # pylint: disable=global-statement + global clip_loaded, adapters_loaded # pylint: disable=global-statement # overrides if hasattr(p, 'ip_adapter_names'): if isinstance(p.ip_adapter_names, str): @@ -274,6 +278,7 @@ def apply(pipe, p: processing.StableDiffusionProcessing, adapter_names=[], adapt subfolders = [adapter['subfolder'] for adapter in adapters] names = [adapter['name'] for adapter in adapters] pipe.load_ip_adapter(repos, subfolder=subfolders, weight_name=names) + adapters_loaded = names if hasattr(p, 'ip_adapter_layers'): pipe.set_ip_adapter_scale(p.ip_adapter_layers) ip_str = ';'.join(adapter_names) + ':' + json.dumps(p.ip_adapter_layers) diff --git a/modules/processing.py b/modules/processing.py index b4839e402..4bc5e3c81 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -179,6 +179,8 @@ def process_images(p: StableDiffusionProcessing) -> Processed: timer.process.record('pre') if shared.cmd_opts.profile: + timer.startup.profile = True + timer.process.profile = True with context_hypertile_vae(p), context_hypertile_unet(p): import torch.profiler # pylint: disable=redefined-outer-name activities=[torch.profiler.ProfilerActivity.CPU] @@ -476,7 +478,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: p.scripts.postprocess(p, processed) timer.process.record('post') if not p.disable_extra_networks: - shared.log.info(f'Processed: images={len(output_images)} its={(p.steps * len(output_images)) / (t1 - t0):.2f} time={t1-t0:.2f} timers={timer.process.dct(min_time=0.02)} memory={memstats.memory_stats()}') + shared.log.info(f'Processed: images={len(output_images)} its={(p.steps * len(output_images)) / (t1 - t0):.2f} time={t1-t0:.2f} timers={timer.process.dct()} memory={memstats.memory_stats()}') devices.torch_gc(force=True) return processed diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index adb047511..581589262 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -4,6 +4,7 @@ import numpy as np import torch import torchvision.transforms.functional as TF +from PIL import Image from modules import shared, devices, processing, sd_models, errors, sd_hijack_hypertile, processing_vae, sd_models_compile, hidiffusion, timer, modelstats, extra_networks from modules.processing_helpers import resize_hires, calculate_base_steps, calculate_hires_steps, calculate_refiner_steps, save_intermediate, update_sampler, is_txt2img, is_refiner_enabled from modules.processing_args import set_pipeline_args @@ -447,6 +448,9 @@ def process_diffusers(p: processing.StableDiffusionProcessing): sd_models_compile.openvino_recompile_model(p, hires=False, refiner=False) # recompile if a parameter changes + if hasattr(p, 'dummy'): + images = [Image.new(mode='RGB', size=(p.width, p.height))] + return images if 'base' not in p.skip: output = process_base(p) else: diff --git a/modules/prompt_parser_diffusers.py b/modules/prompt_parser_diffusers.py index 8c140e0d6..4e31c747a 100644 --- a/modules/prompt_parser_diffusers.py +++ b/modules/prompt_parser_diffusers.py @@ -63,6 +63,8 @@ def __init__(self, prompts, negative_prompts, steps, clip_skip, p): self.positive_schedule = None self.negative_schedule = None self.scheduled_prompt = False + if hasattr(p, 'dummy'): + return earlyout = self.checkcache(p) if earlyout: return diff --git a/modules/shared.py b/modules/shared.py index b1196eba3..371241e0b 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -931,8 +931,11 @@ def get_default_modes(): "extra_networks_styles_sep": OptionInfo("

Styles

", "", gr.HTML), "extra_networks_styles": OptionInfo(True, "Show built-in styles"), + "extra_networks_embed_sep": OptionInfo("

Embeddings

", "", gr.HTML), - "diffusers_convert_embed": OptionInfo(False, "Auto-convert SD15 embeddings to SDXL ", gr.Checkbox, {"visible": native}), + "diffusers_enable_embed": OptionInfo(True, "Enable embeddings support", gr.Checkbox, {"visible": native}), + "diffusers_convert_embed": OptionInfo(False, "Auto-convert SD15 embeddings to SDXL", gr.Checkbox, {"visible": native}), + "extra_networks_wildcard_sep": OptionInfo("

Wildcards

", "", gr.HTML), "wildcards_enabled": OptionInfo(True, "Enable file wildcards support"), })) diff --git a/modules/textual_inversion/textual_inversion.py b/modules/textual_inversion/textual_inversion.py index dd4203a4f..27bb42116 100644 --- a/modules/textual_inversion/textual_inversion.py +++ b/modules/textual_inversion/textual_inversion.py @@ -274,6 +274,8 @@ def load_diffusers_embedding(self, filename: Union[str, List[str]] = None, data: overwrite = bool(data) if not shared.sd_loaded: return + if not shared.opts.diffusers_enable_embed: + return embeddings, skipped = open_embeddings(filename) or convert_bundled(data) for skip in skipped: self.skipped_embeddings[skip.name] = skipped diff --git a/modules/timer.py b/modules/timer.py index 7657ac8e8..43e859140 100644 --- a/modules/timer.py +++ b/modules/timer.py @@ -7,6 +7,7 @@ def __init__(self): self.start = time.time() self.records = {} self.total = 0 + self.profile = False def elapsed(self, reset=True): end = time.time() @@ -27,11 +28,12 @@ def record(self, category=None, extra_time=0, reset=True): category = sys._getframe(1).f_code.co_name # pylint: disable=protected-access if category not in self.records: self.records[category] = 0 - self.records[category] += e + extra_time self.total += e + extra_time def summary(self, min_time=0.05, total=True): + if self.profile: + min_time = -1 res = f"{self.total:.2f} " if total else '' additions = [x for x in self.records.items() if x[1] >= min_time] if not additions: @@ -40,6 +42,8 @@ def summary(self, min_time=0.05, total=True): return res def dct(self, min_time=0.05): + if self.profile: + return {k: round(v, 4) for k, v in self.records.items()} return {k: round(v, 2) for k, v in self.records.items() if v >= min_time} def reset(self): From ee9b0146243d72b4f19c5f45154d332fe1c2b137 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 18 Dec 2024 17:36:42 -0500 Subject: [PATCH 110/249] add ltx-video Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 44 +++++++++-- modules/processing_diffusers.py | 2 + scripts/ltxvideo.py | 130 ++++++++++++++++++++++++++++++++ wiki | 2 +- 4 files changed, 170 insertions(+), 8 deletions(-) create mode 100644 scripts/ltxvideo.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 1931bd2b5..13b831797 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,31 @@ ## Update for 2024-12-18 +### Highlights + +*What's new?* + +While we have several new supported models, workflows and tools, this release is primarily about *quality-of-life improvements*: +- New memory management engine: list of changes that went into this one is too long for here, + but main goal is enabling modern large models to run on standard consumer GPUs + without performance hits typically associated with aggressive memory swapping and needs for constant manual tweaks +- New [documentation website](https://vladmandic.github.io/sdnext-docs/) + with full search and tons of new documentation +- New settings panel with simplified and streamlined configuration + +We've also added support for several new models (see [supported models](https://vladmandic.github.io/sdnext-docs/Model-Support/) for full list): +- [NVLabs Sana](https://huggingface.co/Efficient-Large-Model/Sana_1600M_1024px) +- [Lightricks LTX-Video](https://huggingface.co/Lightricks/LTX-Video) + +And a lot of Control goodies and related goodies +- for SDXL there is new [ProMax](https://huggingface.co/xinsir/controlnet-union-sdxl-1.0), improved *Union* and *Tiling* +- for FLUX.1 there are [Flux Tools](https://blackforestlabs.ai/flux-1-tools/) as well as official *Canny* and *Depth* models and a cool [Redux](https://huggingface.co/black-forest-labs/FLUX.1-Redux-dev) model +- for SD 3.5 there are official *Canny*, *Blur* and *Depth* in addition to existing 3rd party models + +Plus couple of new integrated workflows such as [FreeScale](https://github.com/ali-vilab/FreeScale) and [Style Aligned Image Generation](https://style-aligned-gen.github.io/) + +[README](https://github.com/vladmandic/automatic/blob/master/README.md) | [CHANGELOG](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) | [Docs](https://vladmandic.github.io/sdnext-docs/) | [WiKi](https://github.com/vladmandic/automatic/wiki) | [Discord](https://discord.com/invite/sd-next-federal-batch-inspectors-1101998836328697867) + ### New models and integrations - [NVLabs Sana](https://huggingface.co/Efficient-Large-Model/Sana_1600M_1024px) @@ -13,6 +38,13 @@ *reference values*: sampler: default (or any flow-match variant), width/height: 1024, guidance scale: 4.5 *note* like other LLM-based text-encoders, sana prefers long and descriptive prompts any short prompt below 300 characters will be auto-expanded using built in Gemma LLM before encoding while long prompts will be passed as-is +- **ControlNet** + - improved support for **Union** controlnets with granular control mode type + - added support for latest [Xinsir ProMax](https://huggingface.co/xinsir/controlnet-union-sdxl-1.0) all-in-one controlnet + - added support for multiple **Tiling** controlnets, for example [Xinsir Tile](https://huggingface.co/xinsir/controlnet-tile-sdxl-1.0) + *note*: when selecting tiles in control settings, you can also specify non-square ratios + in which case it will use context-aware image resize to maintain overall composition + *note*: available tiling options can be set in settings -> control - [Flux Tools](https://blackforestlabs.ai/flux-1-tools/) **Redux** is actually a tool, **Fill** is inpaint/outpaint optimized version of *Flux-dev* **Canny** & **Depth** are optimized versions of *Flux-dev* for their respective tasks: they are *not* ControlNets that work on top of a model @@ -36,6 +68,11 @@ both **Depth** and **Canny** LoRAs are available in standard control menus - [StabilityAI SD35 ControlNets](https://huggingface.co/stabilityai/stable-diffusion-3.5-controlnets) - In addition to previously released `InstantX` and `Alimama`, we now have *official* ones from StabilityAI +- [Lightricks LTX-Video](https://huggingface.co/Lightricks/LTX-Video) + basic support for LTX-Video for text-to-video and image-to-video + to use, select in *scripts -> ltx-video* + *note* you may need to enable sequential offload for maximum gpu memory savings + *note* ltx-video requires very long and descriptive prompt, see original link for examples - [Style Aligned Image Generation](https://style-aligned-gen.github.io/) enable in scripts, compatible with sd-xl enter multiple prompts in prompt field separated by new line @@ -47,13 +84,6 @@ run iterative generation of images at different scales to achieve better results can render 4k sdxl images *note*: disable live preview to avoid memory issues when generating large images -- **ControlNet** - - improved support for **Union** controlnets with granular control mode type - - added support for latest [Xinsir ProMax](https://huggingface.co/xinsir/controlnet-union-sdxl-1.0) all-in-one controlnet - - added support for multiple **Tiling** controlnets, for example [Xinsir Tile](https://huggingface.co/xinsir/controlnet-tile-sdxl-1.0) - *note*: when selecting tiles in control settings, you can also specify non-square ratios - in which case it will use context-aware image resize to maintain overall composition - *note*: available tiling options can be set in settings -> control ### UI and workflow improvements diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 581589262..d43660ca8 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -355,6 +355,8 @@ def process_decode(p: processing.StableDiffusionProcessing, output): if not hasattr(output, 'images') and hasattr(output, 'frames'): shared.log.debug(f'Generated: frames={len(output.frames[0])}') output.images = output.frames[0] + if output.images is not None and len(output.images) > 0 and isinstance(output.images[0], Image.Image): + return output.images model = shared.sd_model if not is_refiner_enabled(p) else shared.sd_refiner if not hasattr(model, 'vae'): if hasattr(model, 'pipe') and hasattr(model.pipe, 'vae'): diff --git a/scripts/ltxvideo.py b/scripts/ltxvideo.py new file mode 100644 index 000000000..54e2685a8 --- /dev/null +++ b/scripts/ltxvideo.py @@ -0,0 +1,130 @@ +import time +import torch +import gradio as gr +import diffusers +from modules import scripts, processing, shared, images, devices, sd_models, sd_checkpoint + + +repo_id = 'a-r-r-o-w/LTX-Video-diffusers' +presets = [ + {"label": "custom", "width": 0, "height": 0, "num_frames": 0}, + {"label": "1216x704, 41 frames", "width": 1216, "height": 704, "num_frames": 41}, + {"label": "1088x704, 49 frames", "width": 1088, "height": 704, "num_frames": 49}, + {"label": "1056x640, 57 frames", "width": 1056, "height": 640, "num_frames": 57}, + {"label": "992x608, 65 frames", "width": 992, "height": 608, "num_frames": 65}, + {"label": "896x608, 73 frames", "width": 896, "height": 608, "num_frames": 73}, + {"label": "896x544, 81 frames", "width": 896, "height": 544, "num_frames": 81}, + {"label": "832x544, 89 frames", "width": 832, "height": 544, "num_frames": 89}, + {"label": "800x512, 97 frames", "width": 800, "height": 512, "num_frames": 97}, + {"label": "768x512, 97 frames", "width": 768, "height": 512, "num_frames": 97}, + {"label": "800x480, 105 frames", "width": 800, "height": 480, "num_frames": 105}, + {"label": "736x480, 113 frames", "width": 736, "height": 480, "num_frames": 113}, + {"label": "704x480, 121 frames", "width": 704, "height": 480, "num_frames": 121}, + {"label": "704x448, 129 frames", "width": 704, "height": 448, "num_frames": 129}, + {"label": "672x448, 137 frames", "width": 672, "height": 448, "num_frames": 137}, + {"label": "640x416, 153 frames", "width": 640, "height": 416, "num_frames": 153}, + {"label": "672x384, 161 frames", "width": 672, "height": 384, "num_frames": 161}, + {"label": "640x384, 169 frames", "width": 640, "height": 384, "num_frames": 169}, + {"label": "608x384, 177 frames", "width": 608, "height": 384, "num_frames": 177}, + {"label": "576x384, 185 frames", "width": 576, "height": 384, "num_frames": 185}, + {"label": "608x352, 193 frames", "width": 608, "height": 352, "num_frames": 193}, + {"label": "576x352, 201 frames", "width": 576, "height": 352, "num_frames": 201}, + {"label": "544x352, 209 frames", "width": 544, "height": 352, "num_frames": 209}, + {"label": "512x352, 225 frames", "width": 512, "height": 352, "num_frames": 225}, + {"label": "512x352, 233 frames", "width": 512, "height": 352, "num_frames": 233}, + {"label": "544x320, 241 frames", "width": 544, "height": 320, "num_frames": 241}, + {"label": "512x320, 249 frames", "width": 512, "height": 320, "num_frames": 249}, + {"label": "512x320, 257 frames", "width": 512, "height": 320, "num_frames": 257}, +] + + +class Script(scripts.Script): + def title(self): + return 'Video: LTX Video' + + def show(self, is_img2img): + return shared.native + + # return signature is array of gradio components + def ui(self, _is_img2img): + def video_type_change(video_type): + return [ + gr.update(visible=video_type != 'None'), + gr.update(visible=video_type == 'GIF' or video_type == 'PNG'), + gr.update(visible=video_type == 'MP4'), + gr.update(visible=video_type == 'MP4'), + ] + def preset_change(preset): + return gr.update(visible=preset == 'custom') + + with gr.Row(): + gr.HTML('  LTX Video
') + with gr.Row(): + preset_name = gr.Dropdown(label='Preset', choices=[p['label'] for p in presets], value='custom') + num_frames = gr.Slider(label='Frames', minimum=9, maximum=257, step=1, value=9) + with gr.Row(): + video_type = gr.Dropdown(label='Video file', choices=['None', 'GIF', 'PNG', 'MP4'], value='None') + duration = gr.Slider(label='Duration', minimum=0.25, maximum=10, step=0.25, value=2, visible=False) + with gr.Row(): + gif_loop = gr.Checkbox(label='Loop', value=True, visible=False) + mp4_pad = gr.Slider(label='Pad frames', minimum=0, maximum=24, step=1, value=1, visible=False) + mp4_interpolate = gr.Slider(label='Interpolate frames', minimum=0, maximum=24, step=1, value=0, visible=False) + preset_name.change(fn=preset_change, inputs=[preset_name], outputs=num_frames) + video_type.change(fn=video_type_change, inputs=[video_type], outputs=[duration, gif_loop, mp4_pad, mp4_interpolate]) + return [preset_name, num_frames, video_type, duration, gif_loop, mp4_pad, mp4_interpolate] + + def run(self, p: processing.StableDiffusionProcessing, preset_name, num_frames, video_type, duration, gif_loop, mp4_pad, mp4_interpolate): # pylint: disable=arguments-differ, unused-argument + # set params + preset = [p for p in presets if p['label'] == preset_name][0] + image = getattr(p, 'init_images', None) + image = None if image is None or len(image) == 0 else image[0] + if p.width == 0 or p.height == 0 and image is not None: + p.width = image.width + p.height = image.height + if preset['label'] != 'custom': + num_frames = preset['num_frames'] + p.width = preset['width'] + p.height = preset['height'] + else: + num_frames = 8 * int(num_frames // 8) + 1 + p.width = 32 * int(p.width // 32) + p.height = 32 * int(p.height // 32) + if image: + image = images.resize_image(resize_mode=2, im=image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') + p.task_args['image'] = image + p.task_args['output_type'] = 'pil' + p.task_args['generator'] = torch.manual_seed(p.seed) + p.task_args['num_frames'] = num_frames + p.sampler_name = 'Default' + p.do_not_save_grid = True + p.ops.append('ltx') + + # load model + cls = diffusers.LTXPipeline if image is None else diffusers.LTXImageToVideoPipeline + diffusers.LTXTransformer3DModel = diffusers.LTXVideoTransformer3DModel + diffusers.AutoencoderKLLTX = diffusers.AutoencoderKLLTXVideo + if shared.sd_model.__class__ != cls: + sd_models.unload_model_weights() + shared.sd_model = cls.from_pretrained( + repo_id, + cache_dir = shared.opts.hfcache_dir, + torch_dtype=devices.dtype, + ) + sd_models.set_diffuser_options(shared.sd_model) + shared.sd_model.sd_checkpoint_info = sd_checkpoint.CheckpointInfo(repo_id) + shared.sd_model.sd_model_hash = None + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) + shared.sd_model.vae.enable_slicing() + shared.sd_model.vae.enable_tiling() + devices.torch_gc(force=True) + shared.log.debug(f'LTX: cls={shared.sd_model.__class__.__name__} preset={preset_name} args={p.task_args}') + + # run processing + t0 = time.time() + processed = processing.process_images(p) + t1 = time.time() + if processed is not None and len(processed.images) > 0: + shared.log.info(f'LTX: frames={len(processed.images)} time={t1-t0:.2f}') + if video_type != 'None': + images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=duration, loop=gif_loop, pad=mp4_pad, interpolate=mp4_interpolate) + return processed diff --git a/wiki b/wiki index 470e75f0c..34ba1df45 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 470e75f0c70a22ed3d65187c70f04c131400b35d +Subproject commit 34ba1df45d17da4ee09a2e5278e384bc1929dd8b From 9ddd4d7994835ee483e62579bcb642e3a48514d3 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 19 Dec 2024 11:34:19 -0500 Subject: [PATCH 111/249] update changelog and cleanup Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 20 +++++++++++--------- extensions-builtin/sdnext-modernui | 2 +- html/locale_en.json | 2 +- modules/postprocess/yolo.py | 3 ++- modules/sd_samplers_common.py | 2 +- modules/shared.py | 1 + modules/ui_control.py | 2 +- 7 files changed, 18 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 13b831797..1ccd8afb2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,32 +1,34 @@ # Change Log for SD.Next -## Update for 2024-12-18 +## Update for 2024-12-19 -### Highlights +### Highlights for 2024-12-19 -*What's new?* +### SD.Next X-mass edition: *What's new?* While we have several new supported models, workflows and tools, this release is primarily about *quality-of-life improvements*: -- New memory management engine: list of changes that went into this one is too long for here, +- New memory management engine: list of changes that went into this one is long: changes to GPU offloading, LoRA loader, system memory management, etc. but main goal is enabling modern large models to run on standard consumer GPUs without performance hits typically associated with aggressive memory swapping and needs for constant manual tweaks - New [documentation website](https://vladmandic.github.io/sdnext-docs/) with full search and tons of new documentation - New settings panel with simplified and streamlined configuration -We've also added support for several new models (see [supported models](https://vladmandic.github.io/sdnext-docs/Model-Support/) for full list): -- [NVLabs Sana](https://huggingface.co/Efficient-Large-Model/Sana_1600M_1024px) -- [Lightricks LTX-Video](https://huggingface.co/Lightricks/LTX-Video) +We've also added support for several new models (see [supported models](https://vladmandic.github.io/sdnext-docs/Model-Support/) for full list) such as [NVLabs Sana](https://huggingface.co/Efficient-Large-Model/Sana_1600M_1024px) and [Lightricks LTX-Video](https://huggingface.co/Lightricks/LTX-Video) -And a lot of Control goodies and related goodies +And a lot of Control and IPAdapter goodies - for SDXL there is new [ProMax](https://huggingface.co/xinsir/controlnet-union-sdxl-1.0), improved *Union* and *Tiling* - for FLUX.1 there are [Flux Tools](https://blackforestlabs.ai/flux-1-tools/) as well as official *Canny* and *Depth* models and a cool [Redux](https://huggingface.co/black-forest-labs/FLUX.1-Redux-dev) model - for SD 3.5 there are official *Canny*, *Blur* and *Depth* in addition to existing 3rd party models Plus couple of new integrated workflows such as [FreeScale](https://github.com/ali-vilab/FreeScale) and [Style Aligned Image Generation](https://style-aligned-gen.github.io/) +And it wouldn't be a X-mass edition custom themes: *Snowflake* and *Elf-Green* + [README](https://github.com/vladmandic/automatic/blob/master/README.md) | [CHANGELOG](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) | [Docs](https://vladmandic.github.io/sdnext-docs/) | [WiKi](https://github.com/vladmandic/automatic/wiki) | [Discord](https://discord.com/invite/sd-next-federal-batch-inspectors-1101998836328697867) +## Details for 2024-12-19 + ### New models and integrations - [NVLabs Sana](https://huggingface.co/Efficient-Large-Model/Sana_1600M_1024px) @@ -174,7 +176,7 @@ Plus couple of new integrated workflows such as [FreeScale](https://github.com/a - lora auto-apply tags remove duplicates - control load model on-demand if not already loaded - taesd limit render to 2024px -- taesd downscale preview to 1024px max +- taesd downscale preview to 1024px max: configurable in settings -> live preview ## Update for 2024-11-21 diff --git a/extensions-builtin/sdnext-modernui b/extensions-builtin/sdnext-modernui index 3008cee4b..3f53ff719 160000 --- a/extensions-builtin/sdnext-modernui +++ b/extensions-builtin/sdnext-modernui @@ -1 +1 @@ -Subproject commit 3008cee4b67bb00f8f1a4fe4510ec27ba92aa418 +Subproject commit 3f53ff719f8024003873dd98bb64f21e9401844c diff --git a/html/locale_en.json b/html/locale_en.json index 3c1ac70a4..0767ccd23 100644 --- a/html/locale_en.json +++ b/html/locale_en.json @@ -223,7 +223,7 @@ {"id":"","label":"System Paths","localized":"","hint":"Settings related to location of various model directories"}, {"id":"","label":"Image Options","localized":"","hint":"Settings related to image format, metadata, and image grids"}, {"id":"","label":"image naming & paths","localized":"","hint":"Settings related to image filenames, and output directories"}, - {"id":"","label":"User Interface Options","localized":"","hint":"Settings related to user interface themes, and Quicksettings list"}, + {"id":"","label":"User Interface","localized":"","hint":"Settings related to user interface themes, and Quicksettings list"}, {"id":"","label":"Live Previews","localized":"","hint":"Settings related to live previews, audio notification, and log view"}, {"id":"","label":"Sampler Settings","localized":"","hint":"Settings related to sampler selection and configuration, and diffuser specific sampler configuration"}, {"id":"","label":"Postprocessing","localized":"","hint":"Settings related to post image generation processing, face restoration, and upscaling"}, diff --git a/modules/postprocess/yolo.py b/modules/postprocess/yolo.py index 7bc259391..8fc203280 100644 --- a/modules/postprocess/yolo.py +++ b/modules/postprocess/yolo.py @@ -300,7 +300,8 @@ def restore(self, np_image, p: processing.StableDiffusionProcessing = None): # combined.save('/tmp/item.png') p.image_mask = Image.fromarray(p.image_mask) - shared.log.debug(f'Detailer processed: models={models_used}') + if len(models_used) > 0: + shared.log.debug(f'Detailer processed: models={models_used}') return np_image def ui(self, tab: str): diff --git a/modules/sd_samplers_common.py b/modules/sd_samplers_common.py index 5297d0bcd..2ac7949a6 100644 --- a/modules/sd_samplers_common.py +++ b/modules/sd_samplers_common.py @@ -62,7 +62,7 @@ def single_sample_to_image(sample, approximation=None): sample = sample * (5 / abs(sample_min)) """ if approximation == 2: # TAESD - if sample.shape[-1] > 128 or sample.shape[-2] > 128: + if shared.opts.live_preview_downscale and (sample.shape[-1] > 128 or sample.shape[-2] > 128): scale = 128 / max(sample.shape[-1], sample.shape[-2]) sample = torch.nn.functional.interpolate(sample.unsqueeze(0), scale_factor=[scale, scale], mode='bilinear', align_corners=False)[0] x_sample = sd_vae_taesd.decode(sample) diff --git a/modules/shared.py b/modules/shared.py index 371241e0b..78ff9dc3e 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -786,6 +786,7 @@ def get_default_modes(): "show_progress_type": OptionInfo("Approximate", "Live preview method", gr.Radio, {"choices": ["Simple", "Approximate", "TAESD", "Full VAE"]}), "live_preview_refresh_period": OptionInfo(500, "Progress update period", gr.Slider, {"minimum": 0, "maximum": 5000, "step": 25}), "live_preview_taesd_layers": OptionInfo(3, "TAESD decode layers", gr.Slider, {"minimum": 1, "maximum": 3, "step": 1}), + "live_preview_downscale": OptionInfo(True, "Downscale high resolution live previews"), "logmonitor_show": OptionInfo(True, "Show log view"), "logmonitor_refresh_period": OptionInfo(5000, "Log view update period", gr.Slider, {"minimum": 0, "maximum": 30000, "step": 25}), "notification_audio_enable": OptionInfo(False, "Play a notification upon completion"), diff --git a/modules/ui_control.py b/modules/ui_control.py index 7baf74d75..cd78313a7 100644 --- a/modules/ui_control.py +++ b/modules/ui_control.py @@ -250,10 +250,10 @@ def create_ui(_blocks: gr.Blocks=None): process_id = gr.Dropdown(label="Processor", choices=processors.list_models(), value='None', elem_id=f'control_unit-{i}-process_name') model_id = gr.Dropdown(label="ControlNet", choices=controlnet.list_models(), value='None', elem_id=f'control_unit-{i}-model_name') ui_common.create_refresh_button(model_id, controlnet.list_models, lambda: {"choices": controlnet.list_models(refresh=True)}, f'refresh_controlnet_models_{i}') + control_mode = gr.Dropdown(label="CN Mode", choices=['default'], value='default', visible=False, elem_id=f'control_unit-{i}-mode') model_strength = gr.Slider(label="CN Strength", minimum=0.01, maximum=2.0, step=0.01, value=1.0, elem_id=f'control_unit-{i}-strength') control_start = gr.Slider(label="CN Start", minimum=0.0, maximum=1.0, step=0.05, value=0, elem_id=f'control_unit-{i}-start') control_end = gr.Slider(label="CN End", minimum=0.0, maximum=1.0, step=0.05, value=1.0, elem_id=f'control_unit-{i}-end') - control_mode = gr.Dropdown(label="CN Mode", choices=['default'], value='default', visible=False, elem_id=f'control_unit-{i}-mode') control_tile = gr.Dropdown(label="CN Tiles", choices=[x.strip() for x in shared.opts.control_tiles.split(',') if 'x' in x], value='1x1', visible=False, elem_id=f'control_unit-{i}-tile') reset_btn = ui_components.ToolButton(value=ui_symbols.reset) image_upload = gr.UploadButton(label=ui_symbols.upload, file_types=['image'], elem_classes=['form', 'gradio-button', 'tool']) From f9fda69970827bcb8f6f16e62def9d77bee82195 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 19 Dec 2024 11:53:41 -0500 Subject: [PATCH 112/249] uninstall wandb Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 1 + installer.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ccd8afb2..9df8c4e1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -177,6 +177,7 @@ And it wouldn't be a X-mass edition custom themes: *Snowflake* and *Elf-Green* - control load model on-demand if not already loaded - taesd limit render to 2024px - taesd downscale preview to 1024px max: configurable in settings -> live preview +- uninstall conflicting `wandb` package ## Update for 2024-11-21 diff --git a/installer.py b/installer.py index 3dd0f13d5..4ba32e4c7 100644 --- a/installer.py +++ b/installer.py @@ -686,6 +686,8 @@ def install_torch_addons(): install('nncf==2.7.0', 'nncf') if opts.get('optimum_quanto_weights', False): install('optimum-quanto==0.2.6', 'optimum-quanto') + if not args.experimental: + uninstall('wandb', quiet=True) if triton_command is not None: install(triton_command, 'triton', quiet=True) From 8c856f6d996f464963586f19a7d792aca5d2f351 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 19 Dec 2024 11:56:23 -0500 Subject: [PATCH 113/249] dont use uv for uninstall Signed-off-by: Vladimir Mandic --- installer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installer.py b/installer.py index 4ba32e4c7..76bd5e9e8 100644 --- a/installer.py +++ b/installer.py @@ -250,7 +250,7 @@ def uninstall(package, quiet = False): if installed(p, p, quiet=True): if not quiet: log.warning(f'Package: {p} uninstall') - res += pip(f"uninstall {p} --yes --quiet", ignore=True, quiet=True) + res += pip(f"uninstall {p} --yes --quiet", ignore=True, quiet=True, uv=False) return res From 82d6916f1abfa9c3bba3c9145e3a18a70f78897e Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 19 Dec 2024 14:52:29 -0500 Subject: [PATCH 114/249] add hunyuan video and mochi video Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 32 +++++++-- scripts/animatediff.py | 2 +- scripts/cogvideo.py | 2 +- scripts/hunyuanvideo.py | 111 ++++++++++++++++++++++++++++++++ scripts/image2video.py | 2 +- scripts/ltxvideo.py | 64 ++++-------------- scripts/mochivideo.py | 85 ++++++++++++++++++++++++ scripts/stablevideodiffusion.py | 2 +- scripts/text2video.py | 2 +- wiki | 2 +- 10 files changed, 242 insertions(+), 62 deletions(-) create mode 100644 scripts/hunyuanvideo.py create mode 100644 scripts/mochivideo.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 9df8c4e1e..df917041b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,8 @@ While we have several new supported models, workflows and tools, this release is with full search and tons of new documentation - New settings panel with simplified and streamlined configuration -We've also added support for several new models (see [supported models](https://vladmandic.github.io/sdnext-docs/Model-Support/) for full list) such as [NVLabs Sana](https://huggingface.co/Efficient-Large-Model/Sana_1600M_1024px) and [Lightricks LTX-Video](https://huggingface.co/Lightricks/LTX-Video) +We've also added support for several new models (see [supported models](https://vladmandic.github.io/sdnext-docs/Model-Support/) for full list) such as highly anticipated [NVLabs Sana](https://huggingface.co/Efficient-Large-Model/Sana_1600M_1024px) +And several new video models: [Lightricks LTX-Video](https://huggingface.co/Lightricks/LTX-Video), [Hunyuan Video](https://huggingface.co/tencent/HunyuanVideo) and [Genmo Mochi.1 Preview](https://huggingface.co/genmo/mochi-1-preview) And a lot of Control and IPAdapter goodies - for SDXL there is new [ProMax](https://huggingface.co/xinsir/controlnet-union-sdxl-1.0), improved *Union* and *Tiling* @@ -70,11 +71,6 @@ And it wouldn't be a X-mass edition custom themes: *Snowflake* and *Elf-Green* both **Depth** and **Canny** LoRAs are available in standard control menus - [StabilityAI SD35 ControlNets](https://huggingface.co/stabilityai/stable-diffusion-3.5-controlnets) - In addition to previously released `InstantX` and `Alimama`, we now have *official* ones from StabilityAI -- [Lightricks LTX-Video](https://huggingface.co/Lightricks/LTX-Video) - basic support for LTX-Video for text-to-video and image-to-video - to use, select in *scripts -> ltx-video* - *note* you may need to enable sequential offload for maximum gpu memory savings - *note* ltx-video requires very long and descriptive prompt, see original link for examples - [Style Aligned Image Generation](https://style-aligned-gen.github.io/) enable in scripts, compatible with sd-xl enter multiple prompts in prompt field separated by new line @@ -87,6 +83,30 @@ And it wouldn't be a X-mass edition custom themes: *Snowflake* and *Elf-Green* can render 4k sdxl images *note*: disable live preview to avoid memory issues when generating large images +### Video models + +- [Lightricks LTX-Video](https://huggingface.co/Lightricks/LTX-Video) + model size: 27.75gb + support for text-to-video and image-to-video, to use, select in *scripts -> ltx-video* + *refrence values*: steps 50, width 704, height 512, frames 161, guidance scale 3.0 +- [Hunyuan Video](https://huggingface.co/tencent/HunyuanVideo) + model size: 40.92gb + support for text-to-video, to use, select in *scripts -> hunyuan video* + *refrence values*: steps 50, width 1280, height 720, frames 129, guidance scale 6.0 +- [Genmo Mochi.1 Preview](https://huggingface.co/genmo/mochi-1-preview) + support for text-to-video, to use, select in *scripts -> mochi.1 video* + *refrence values*: steps 64, width 848, height 480, frames 19, guidance scale 4.5 + +*Notes*: +- all video models are very large and resource intensive! + any use on gpus below 16gb and systems below 48gb ram is experimental at best +- sdnext support for video models is relatively basic with further optimizations pending community interest + any future optimizations would likely have to go into partial loading and excecution instead of offloading inactive parts of the model +- new video models use generic llms for prompting and due to that requires very long and descriptive prompt +- you may need to enable sequential offload for maximum gpu memory savings +- optionally enable pre-quantization using bnb for additional memory savings +- reduce number of frames and/or resolution to reduce memory usage + ### UI and workflow improvements - **Docs**: diff --git a/scripts/animatediff.py b/scripts/animatediff.py index 91db60915..6c29f3fa5 100644 --- a/scripts/animatediff.py +++ b/scripts/animatediff.py @@ -250,7 +250,7 @@ def run(self, p: processing.StableDiffusionProcessing, adapter_index, frames, lo processing.fix_seed(p) p.extra_generation_params['AnimateDiff'] = loaded_adapter p.do_not_save_grid = True - p.ops.append('animatediff') + p.ops.append('video') p.task_args['generator'] = None p.task_args['num_frames'] = frames p.task_args['num_inference_steps'] = p.steps diff --git a/scripts/cogvideo.py b/scripts/cogvideo.py index a5efcd3e6..e689a5e3f 100644 --- a/scripts/cogvideo.py +++ b/scripts/cogvideo.py @@ -202,7 +202,7 @@ def run(self, p: processing.StableDiffusionProcessing, model, sampler, frames, g p.extra_generation_params['CogVideoX'] = model p.do_not_save_grid = True if 'animatediff' not in p.ops: - p.ops.append('cogvideox') + p.ops.append('video') if override: p.width = 720 p.height = 480 diff --git a/scripts/hunyuanvideo.py b/scripts/hunyuanvideo.py new file mode 100644 index 000000000..b94c8b8f8 --- /dev/null +++ b/scripts/hunyuanvideo.py @@ -0,0 +1,111 @@ +import time +import torch +import gradio as gr +import diffusers +from modules import scripts, processing, shared, images, devices, sd_models, sd_checkpoint, model_quant + + +repo_id = 'tencent/HunyuanVideo' +""" +prompt_template = { # default + "template": ( + "<|start_header_id|>system<|end_header_id|>\n\nDescribe the video by detailing the following aspects: " + "1. The main content and theme of the video." + "2. The color, shape, size, texture, quantity, text, and spatial relationships of the contents, including objects, people, and anything else." + "3. Actions, events, behaviors temporal relationships, physical movement changes of the contents." + "4. Background environment, light, style, atmosphere, and qualities." + "5. Camera angles, movements, and transitions used in the video." + "6. Thematic and aesthetic concepts associated with the scene, i.e. realistic, futuristic, fairy tale, etc<|eot_id|>" + "<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|>" + ), + "crop_start": 95, +} +""" + + +class Script(scripts.Script): + def title(self): + return 'Video: Hunyuan Video' + + def show(self, is_img2img): + return not is_img2img if shared.native else False + + # return signature is array of gradio components + def ui(self, _is_img2img): + def video_type_change(video_type): + return [ + gr.update(visible=video_type != 'None'), + gr.update(visible=video_type == 'GIF' or video_type == 'PNG'), + gr.update(visible=video_type == 'MP4'), + gr.update(visible=video_type == 'MP4'), + ] + + with gr.Row(): + gr.HTML('  Hunyuan Video
') + with gr.Row(): + num_frames = gr.Slider(label='Frames', minimum=9, maximum=257, step=1, value=45) + with gr.Row(): + video_type = gr.Dropdown(label='Video file', choices=['None', 'GIF', 'PNG', 'MP4'], value='None') + duration = gr.Slider(label='Duration', minimum=0.25, maximum=10, step=0.25, value=2, visible=False) + with gr.Row(): + gif_loop = gr.Checkbox(label='Loop', value=True, visible=False) + mp4_pad = gr.Slider(label='Pad frames', minimum=0, maximum=24, step=1, value=1, visible=False) + mp4_interpolate = gr.Slider(label='Interpolate frames', minimum=0, maximum=24, step=1, value=0, visible=False) + video_type.change(fn=video_type_change, inputs=[video_type], outputs=[duration, gif_loop, mp4_pad, mp4_interpolate]) + return [num_frames, video_type, duration, gif_loop, mp4_pad, mp4_interpolate] + + def run(self, p: processing.StableDiffusionProcessing, num_frames, video_type, duration, gif_loop, mp4_pad, mp4_interpolate): # pylint: disable=arguments-differ, unused-argument + # set params + num_frames = int(num_frames) + p.width = 32 * int(p.width // 32) + p.height = 32 * int(p.height // 32) + p.task_args['output_type'] = 'pil' + p.task_args['generator'] = torch.manual_seed(p.seed) + p.task_args['num_frames'] = num_frames + # p.task_args['prompt_template'] = prompt_template + p.sampler_name = 'Default' + p.do_not_save_grid = True + p.ops.append('video') + + # load model + cls = diffusers.HunyuanVideoPipeline + if shared.sd_model.__class__ != cls: + sd_models.unload_model_weights() + kwargs = {} + kwargs = model_quant.create_bnb_config(kwargs) + kwargs = model_quant.create_ao_config(kwargs) + transformer = diffusers.HunyuanVideoTransformer3DModel.from_pretrained( + repo_id, + subfolder="transformer", + torch_dtype=devices.dtype, + revision="refs/pr/18", + cache_dir = shared.opts.hfcache_dir, + **kwargs + ) + shared.sd_model = cls.from_pretrained( + repo_id, + transformer=transformer, + revision="refs/pr/18", + cache_dir = shared.opts.hfcache_dir, + torch_dtype=devices.dtype, + **kwargs + ) + shared.sd_model.scheduler._shift = 7.0 # pylint: disable=protected-access + sd_models.set_diffuser_options(shared.sd_model) + shared.sd_model.sd_checkpoint_info = sd_checkpoint.CheckpointInfo(repo_id) + shared.sd_model.sd_model_hash = None + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) + shared.sd_model.vae.enable_slicing() + shared.sd_model.vae.enable_tiling() + devices.torch_gc(force=True) + shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} args={p.task_args}') + + # run processing + t0 = time.time() + processed = processing.process_images(p) + t1 = time.time() + if processed is not None and len(processed.images) > 0: + shared.log.info(f'Video: frames={len(processed.images)} time={t1-t0:.2f}') + if video_type != 'None': + images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=duration, loop=gif_loop, pad=mp4_pad, interpolate=mp4_interpolate) + return processed diff --git a/scripts/image2video.py b/scripts/image2video.py index 5e08922ee..ad6615f67 100644 --- a/scripts/image2video.py +++ b/scripts/image2video.py @@ -73,7 +73,7 @@ def run(self, p: processing.StableDiffusionProcessing, model_name, num_frames, v model = [m for m in MODELS if m['name'] == model_name][0] repo_id = model['url'] shared.log.debug(f'Image2Video: model={model_name} frames={num_frames}, video={video_type} duration={duration} loop={gif_loop} pad={mp4_pad} interpolate={mp4_interpolate}') - p.ops.append('image2video') + p.ops.append('video') p.do_not_save_grid = True orig_pipeline = shared.sd_model diff --git a/scripts/ltxvideo.py b/scripts/ltxvideo.py index 54e2685a8..50530563a 100644 --- a/scripts/ltxvideo.py +++ b/scripts/ltxvideo.py @@ -2,40 +2,10 @@ import torch import gradio as gr import diffusers -from modules import scripts, processing, shared, images, devices, sd_models, sd_checkpoint +from modules import scripts, processing, shared, images, devices, sd_models, sd_checkpoint, model_quant repo_id = 'a-r-r-o-w/LTX-Video-diffusers' -presets = [ - {"label": "custom", "width": 0, "height": 0, "num_frames": 0}, - {"label": "1216x704, 41 frames", "width": 1216, "height": 704, "num_frames": 41}, - {"label": "1088x704, 49 frames", "width": 1088, "height": 704, "num_frames": 49}, - {"label": "1056x640, 57 frames", "width": 1056, "height": 640, "num_frames": 57}, - {"label": "992x608, 65 frames", "width": 992, "height": 608, "num_frames": 65}, - {"label": "896x608, 73 frames", "width": 896, "height": 608, "num_frames": 73}, - {"label": "896x544, 81 frames", "width": 896, "height": 544, "num_frames": 81}, - {"label": "832x544, 89 frames", "width": 832, "height": 544, "num_frames": 89}, - {"label": "800x512, 97 frames", "width": 800, "height": 512, "num_frames": 97}, - {"label": "768x512, 97 frames", "width": 768, "height": 512, "num_frames": 97}, - {"label": "800x480, 105 frames", "width": 800, "height": 480, "num_frames": 105}, - {"label": "736x480, 113 frames", "width": 736, "height": 480, "num_frames": 113}, - {"label": "704x480, 121 frames", "width": 704, "height": 480, "num_frames": 121}, - {"label": "704x448, 129 frames", "width": 704, "height": 448, "num_frames": 129}, - {"label": "672x448, 137 frames", "width": 672, "height": 448, "num_frames": 137}, - {"label": "640x416, 153 frames", "width": 640, "height": 416, "num_frames": 153}, - {"label": "672x384, 161 frames", "width": 672, "height": 384, "num_frames": 161}, - {"label": "640x384, 169 frames", "width": 640, "height": 384, "num_frames": 169}, - {"label": "608x384, 177 frames", "width": 608, "height": 384, "num_frames": 177}, - {"label": "576x384, 185 frames", "width": 576, "height": 384, "num_frames": 185}, - {"label": "608x352, 193 frames", "width": 608, "height": 352, "num_frames": 193}, - {"label": "576x352, 201 frames", "width": 576, "height": 352, "num_frames": 201}, - {"label": "544x352, 209 frames", "width": 544, "height": 352, "num_frames": 209}, - {"label": "512x352, 225 frames", "width": 512, "height": 352, "num_frames": 225}, - {"label": "512x352, 233 frames", "width": 512, "height": 352, "num_frames": 233}, - {"label": "544x320, 241 frames", "width": 544, "height": 320, "num_frames": 241}, - {"label": "512x320, 249 frames", "width": 512, "height": 320, "num_frames": 249}, - {"label": "512x320, 257 frames", "width": 512, "height": 320, "num_frames": 257}, -] class Script(scripts.Script): @@ -54,14 +24,11 @@ def video_type_change(video_type): gr.update(visible=video_type == 'MP4'), gr.update(visible=video_type == 'MP4'), ] - def preset_change(preset): - return gr.update(visible=preset == 'custom') with gr.Row(): gr.HTML('  LTX Video
') with gr.Row(): - preset_name = gr.Dropdown(label='Preset', choices=[p['label'] for p in presets], value='custom') - num_frames = gr.Slider(label='Frames', minimum=9, maximum=257, step=1, value=9) + num_frames = gr.Slider(label='Frames', minimum=9, maximum=257, step=1, value=41) with gr.Row(): video_type = gr.Dropdown(label='Video file', choices=['None', 'GIF', 'PNG', 'MP4'], value='None') duration = gr.Slider(label='Duration', minimum=0.25, maximum=10, step=0.25, value=2, visible=False) @@ -69,26 +36,19 @@ def preset_change(preset): gif_loop = gr.Checkbox(label='Loop', value=True, visible=False) mp4_pad = gr.Slider(label='Pad frames', minimum=0, maximum=24, step=1, value=1, visible=False) mp4_interpolate = gr.Slider(label='Interpolate frames', minimum=0, maximum=24, step=1, value=0, visible=False) - preset_name.change(fn=preset_change, inputs=[preset_name], outputs=num_frames) video_type.change(fn=video_type_change, inputs=[video_type], outputs=[duration, gif_loop, mp4_pad, mp4_interpolate]) - return [preset_name, num_frames, video_type, duration, gif_loop, mp4_pad, mp4_interpolate] + return [num_frames, video_type, duration, gif_loop, mp4_pad, mp4_interpolate] - def run(self, p: processing.StableDiffusionProcessing, preset_name, num_frames, video_type, duration, gif_loop, mp4_pad, mp4_interpolate): # pylint: disable=arguments-differ, unused-argument + def run(self, p: processing.StableDiffusionProcessing, num_frames, video_type, duration, gif_loop, mp4_pad, mp4_interpolate): # pylint: disable=arguments-differ, unused-argument # set params - preset = [p for p in presets if p['label'] == preset_name][0] image = getattr(p, 'init_images', None) image = None if image is None or len(image) == 0 else image[0] if p.width == 0 or p.height == 0 and image is not None: p.width = image.width p.height = image.height - if preset['label'] != 'custom': - num_frames = preset['num_frames'] - p.width = preset['width'] - p.height = preset['height'] - else: - num_frames = 8 * int(num_frames // 8) + 1 - p.width = 32 * int(p.width // 32) - p.height = 32 * int(p.height // 32) + num_frames = 8 * int(num_frames // 8) + 1 + p.width = 32 * int(p.width // 32) + p.height = 32 * int(p.height // 32) if image: image = images.resize_image(resize_mode=2, im=image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') p.task_args['image'] = image @@ -97,7 +57,7 @@ def run(self, p: processing.StableDiffusionProcessing, preset_name, num_frames, p.task_args['num_frames'] = num_frames p.sampler_name = 'Default' p.do_not_save_grid = True - p.ops.append('ltx') + p.ops.append('video') # load model cls = diffusers.LTXPipeline if image is None else diffusers.LTXImageToVideoPipeline @@ -105,10 +65,14 @@ def run(self, p: processing.StableDiffusionProcessing, preset_name, num_frames, diffusers.AutoencoderKLLTX = diffusers.AutoencoderKLLTXVideo if shared.sd_model.__class__ != cls: sd_models.unload_model_weights() + kwargs = {} + kwargs = model_quant.create_bnb_config(kwargs) + kwargs = model_quant.create_ao_config(kwargs) shared.sd_model = cls.from_pretrained( repo_id, cache_dir = shared.opts.hfcache_dir, torch_dtype=devices.dtype, + **kwargs ) sd_models.set_diffuser_options(shared.sd_model) shared.sd_model.sd_checkpoint_info = sd_checkpoint.CheckpointInfo(repo_id) @@ -117,14 +81,14 @@ def run(self, p: processing.StableDiffusionProcessing, preset_name, num_frames, shared.sd_model.vae.enable_slicing() shared.sd_model.vae.enable_tiling() devices.torch_gc(force=True) - shared.log.debug(f'LTX: cls={shared.sd_model.__class__.__name__} preset={preset_name} args={p.task_args}') + shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} args={p.task_args}') # run processing t0 = time.time() processed = processing.process_images(p) t1 = time.time() if processed is not None and len(processed.images) > 0: - shared.log.info(f'LTX: frames={len(processed.images)} time={t1-t0:.2f}') + shared.log.info(f'Video: frames={len(processed.images)} time={t1-t0:.2f}') if video_type != 'None': images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=duration, loop=gif_loop, pad=mp4_pad, interpolate=mp4_interpolate) return processed diff --git a/scripts/mochivideo.py b/scripts/mochivideo.py new file mode 100644 index 000000000..f85616a5e --- /dev/null +++ b/scripts/mochivideo.py @@ -0,0 +1,85 @@ +import time +import torch +import gradio as gr +import diffusers +from modules import scripts, processing, shared, images, devices, sd_models, sd_checkpoint, model_quant + + +repo_id = 'genmo/mochi-1-preview' + + +class Script(scripts.Script): + def title(self): + return 'Video: Mochi.1 Video' + + def show(self, is_img2img): + return not is_img2img if shared.native else False + + # return signature is array of gradio components + def ui(self, _is_img2img): + def video_type_change(video_type): + return [ + gr.update(visible=video_type != 'None'), + gr.update(visible=video_type == 'GIF' or video_type == 'PNG'), + gr.update(visible=video_type == 'MP4'), + gr.update(visible=video_type == 'MP4'), + ] + + with gr.Row(): + gr.HTML('  Mochi.1 Video
') + with gr.Row(): + num_frames = gr.Slider(label='Frames', minimum=9, maximum=257, step=1, value=45) + with gr.Row(): + video_type = gr.Dropdown(label='Video file', choices=['None', 'GIF', 'PNG', 'MP4'], value='None') + duration = gr.Slider(label='Duration', minimum=0.25, maximum=10, step=0.25, value=2, visible=False) + with gr.Row(): + gif_loop = gr.Checkbox(label='Loop', value=True, visible=False) + mp4_pad = gr.Slider(label='Pad frames', minimum=0, maximum=24, step=1, value=1, visible=False) + mp4_interpolate = gr.Slider(label='Interpolate frames', minimum=0, maximum=24, step=1, value=0, visible=False) + video_type.change(fn=video_type_change, inputs=[video_type], outputs=[duration, gif_loop, mp4_pad, mp4_interpolate]) + return [num_frames, video_type, duration, gif_loop, mp4_pad, mp4_interpolate] + + def run(self, p: processing.StableDiffusionProcessing, num_frames, video_type, duration, gif_loop, mp4_pad, mp4_interpolate): # pylint: disable=arguments-differ, unused-argument + # set params + num_frames = int(num_frames // 8) + p.width = 32 * int(p.width // 32) + p.height = 32 * int(p.height // 32) + p.task_args['output_type'] = 'pil' + p.task_args['generator'] = torch.manual_seed(p.seed) + p.task_args['num_frames'] = num_frames + p.sampler_name = 'Default' + p.do_not_save_grid = True + p.ops.append('video') + + # load model + cls = diffusers.MochiPipeline + if shared.sd_model.__class__ != cls: + sd_models.unload_model_weights() + kwargs = {} + kwargs = model_quant.create_bnb_config(kwargs) + kwargs = model_quant.create_ao_config(kwargs) + shared.sd_model = cls.from_pretrained( + repo_id, + cache_dir = shared.opts.hfcache_dir, + torch_dtype=devices.dtype, + **kwargs + ) + shared.sd_model.scheduler._shift = 7.0 # pylint: disable=protected-access + sd_models.set_diffuser_options(shared.sd_model) + shared.sd_model.sd_checkpoint_info = sd_checkpoint.CheckpointInfo(repo_id) + shared.sd_model.sd_model_hash = None + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) + shared.sd_model.vae.enable_slicing() + shared.sd_model.vae.enable_tiling() + devices.torch_gc(force=True) + shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} args={p.task_args}') + + # run processing + t0 = time.time() + processed = processing.process_images(p) + t1 = time.time() + if processed is not None and len(processed.images) > 0: + shared.log.info(f'Video: frames={len(processed.images)} time={t1-t0:.2f}') + if video_type != 'None': + images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=duration, loop=gif_loop, pad=mp4_pad, interpolate=mp4_interpolate) + return processed diff --git a/scripts/stablevideodiffusion.py b/scripts/stablevideodiffusion.py index cbf2ce003..f8da35b23 100644 --- a/scripts/stablevideodiffusion.py +++ b/scripts/stablevideodiffusion.py @@ -81,7 +81,7 @@ def run(self, p: processing.StableDiffusionProcessing, model, num_frames, overri p.width = 1024 p.height = 576 image = images.resize_image(resize_mode=2, im=image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') - p.ops.append('svd') + p.ops.append('video') p.do_not_save_grid = True p.init_images = [image] p.sampler_name = 'Default' # svd does not support non-default sampler diff --git a/scripts/text2video.py b/scripts/text2video.py index dc4c44cac..c7b3d1c05 100644 --- a/scripts/text2video.py +++ b/scripts/text2video.py @@ -87,7 +87,7 @@ def run(self, p: processing.StableDiffusionProcessing, model_name, use_default, shared.opts.sd_model_checkpoint = checkpoint.name sd_models.reload_model_weights(op='model') - p.ops.append('text2video') + p.ops.append('video') p.do_not_save_grid = True if use_default: p.task_args['num_frames'] = model['params'][0] diff --git a/wiki b/wiki index 34ba1df45..a6c10ce38 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 34ba1df45d17da4ee09a2e5278e384bc1929dd8b +Subproject commit a6c10ce38ef1da4d47cd68ad0aa8552d2d62c943 From cba014e0090c979d8ab6d03ce69e42c99be27c67 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 19 Dec 2024 15:01:38 -0500 Subject: [PATCH 115/249] update wiki Signed-off-by: Vladimir Mandic --- wiki | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wiki b/wiki index a6c10ce38..8db442124 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit a6c10ce38ef1da4d47cd68ad0aa8552d2d62c943 +Subproject commit 8db44212407343c1855d8811efb61f6e69bd4caa From 042e5bff29af0f2207d07a83ace5ab592a8f6e83 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 20 Dec 2024 10:22:42 -0500 Subject: [PATCH 116/249] add sd35-ipadapter and more balanced offload optimizations Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 16 ++-- installer.py | 4 +- modules/devices.py | 18 ++-- modules/ipadapter.py | 193 +++++++++++++++++++++++++----------------- modules/processing.py | 2 +- modules/sd_models.py | 9 +- scripts/flux_tools.py | 3 +- 7 files changed, 148 insertions(+), 97 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index df917041b..d2d70f3e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ ### Highlights for 2024-12-19 -### SD.Next X-mass edition: *What's new?* +### SD.Next Xmass edition: *What's new?* While we have several new supported models, workflows and tools, this release is primarily about *quality-of-life improvements*: - New memory management engine: list of changes that went into this one is long: changes to GPU offloading, LoRA loader, system memory management, etc. @@ -15,18 +15,18 @@ While we have several new supported models, workflows and tools, this release is - New settings panel with simplified and streamlined configuration We've also added support for several new models (see [supported models](https://vladmandic.github.io/sdnext-docs/Model-Support/) for full list) such as highly anticipated [NVLabs Sana](https://huggingface.co/Efficient-Large-Model/Sana_1600M_1024px) -And several new video models: [Lightricks LTX-Video](https://huggingface.co/Lightricks/LTX-Video), [Hunyuan Video](https://huggingface.co/tencent/HunyuanVideo) and [Genmo Mochi.1 Preview](https://huggingface.co/genmo/mochi-1-preview) +And several new SOTA video models: [Lightricks LTX-Video](https://huggingface.co/Lightricks/LTX-Video), [Hunyuan Video](https://huggingface.co/tencent/HunyuanVideo) and [Genmo Mochi.1 Preview](https://huggingface.co/genmo/mochi-1-preview) And a lot of Control and IPAdapter goodies -- for SDXL there is new [ProMax](https://huggingface.co/xinsir/controlnet-union-sdxl-1.0), improved *Union* and *Tiling* -- for FLUX.1 there are [Flux Tools](https://blackforestlabs.ai/flux-1-tools/) as well as official *Canny* and *Depth* models and a cool [Redux](https://huggingface.co/black-forest-labs/FLUX.1-Redux-dev) model -- for SD 3.5 there are official *Canny*, *Blur* and *Depth* in addition to existing 3rd party models +- for **SDXL** there is new [ProMax](https://huggingface.co/xinsir/controlnet-union-sdxl-1.0), improved *Union* and *Tiling* models +- for **FLUX.1** there are [Flux Tools](https://blackforestlabs.ai/flux-1-tools/) as well as official *Canny* and *Depth* models and a cool [Redux](https://huggingface.co/black-forest-labs/FLUX.1-Redux-dev) model +- for **SD3.5** there are official *Canny*, *Blur* and *Depth* models in addition to existing 3rd party models as well as [InstantX](https://huggingface.co/InstantX/SD3.5-Large-IP-Adapter) IP-adapter Plus couple of new integrated workflows such as [FreeScale](https://github.com/ali-vilab/FreeScale) and [Style Aligned Image Generation](https://style-aligned-gen.github.io/) -And it wouldn't be a X-mass edition custom themes: *Snowflake* and *Elf-Green* +And it wouldn't be a Xmass edition without couple of custom themes: *Snowflake* and *Elf-Green*! -[README](https://github.com/vladmandic/automatic/blob/master/README.md) | [CHANGELOG](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) | [Docs](https://vladmandic.github.io/sdnext-docs/) | [WiKi](https://github.com/vladmandic/automatic/wiki) | [Discord](https://discord.com/invite/sd-next-federal-batch-inspectors-1101998836328697867) +[ReadMe](https://github.com/vladmandic/automatic/blob/master/README.md) | [ChangeLog](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) | [Docs](https://vladmandic.github.io/sdnext-docs/) | [WiKi](https://github.com/vladmandic/automatic/wiki) | [Discord](https://discord.com/invite/sd-next-federal-batch-inspectors-1101998836328697867) ## Details for 2024-12-19 @@ -136,6 +136,7 @@ And it wouldn't be a X-mass edition custom themes: *Snowflake* and *Elf-Green* - balanced offload: add both high and low watermark, defaults as below `0.25` for low-watermark: skip offload if memory usage is below 25% `0.70` high-watermark: must offload if memory usage is above 70% + - balanced offload will attempt to run offload as non-blocking and force gc at the end - change-in-behavior: low-end systems, triggered by either `lowvrwam` or by detection of <=4GB will use *sequential offload* all other systems use *balanced offload* by default (can be changed in settings) @@ -198,6 +199,7 @@ And it wouldn't be a X-mass edition custom themes: *Snowflake* and *Elf-Green* - taesd limit render to 2024px - taesd downscale preview to 1024px max: configurable in settings -> live preview - uninstall conflicting `wandb` package +- dont skip diffusers version check if quick is specified ## Update for 2024-11-21 diff --git a/installer.py b/installer.py index 76bd5e9e8..f1360541f 100644 --- a/installer.py +++ b/installer.py @@ -457,9 +457,9 @@ def check_python(supported_minors=[9, 10, 11, 12], reason=None): # check diffusers version def check_diffusers(): - if args.skip_all or args.skip_requirements: + if args.skip_all or args.skip_git: return - sha = '862a7d5038c1c53641ffcab146a7eeb5ab683656' # diffusers commit hash + sha = 'b64ca6c11cbc1644c22f1dae441c8124d588bb14' # diffusers commit hash pkg = pkg_resources.working_set.by_key.get('diffusers', None) minor = int(pkg.version.split('.')[1] if pkg is not None else 0) cur = opts.get('diffusers_version', '') if minor > 0 else '' diff --git a/modules/devices.py b/modules/devices.py index 3f1439fb7..949fab4aa 100644 --- a/modules/devices.py +++ b/modules/devices.py @@ -186,7 +186,7 @@ def get_device_for(task): # pylint: disable=unused-argument return get_optimal_device() -def torch_gc(force=False, fast=False): +def torch_gc(force:bool=False, fast:bool=False, reason:str=None): def get_stats(): mem_dict = memstats.memory_stats() gpu_dict = mem_dict.get('gpu', {}) @@ -207,15 +207,21 @@ def get_stats(): from modules.shared import cmd_opts t0 = time.time() - gpu, used_gpu, ram, used_ram, oom = get_stats() + gpu, used_gpu, ram, _used_ram, oom = get_stats() threshold = 0 if (cmd_opts.lowvram and not cmd_opts.use_zluda) else opts.torch_gc_threshold collected = 0 - if force or threshold == 0 or used_gpu >= threshold or used_ram >= threshold: + if reason is None and force: + reason='force' + if threshold == 0 or used_gpu >= threshold: force = True + if reason is None: + reason = 'threshold' if oom > previous_oom: previous_oom = oom log.warning(f'Torch GPU out-of-memory error: {memstats.memory_stats()}') force = True + if reason is None: + reason = 'oom' if force: # actual gc collected = gc.collect() if not fast else 0 # python gc @@ -237,10 +243,10 @@ def get_stats(): new_gpu, new_used_gpu, new_ram, new_used_ram, oom = get_stats() before = { 'gpu': gpu, 'ram': ram } after = { 'gpu': new_gpu, 'ram': new_ram, 'oom': oom } - utilization = { 'gpu': new_used_gpu, 'ram': new_used_ram, 'threshold': threshold } - results = { 'saved': round(gpu - new_gpu, 2), 'collected': collected } + utilization = { 'gpu': new_used_gpu, 'ram': new_used_ram } + results = { 'gpu': round(gpu - new_gpu, 2), 'py': collected } fn = f'{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access - log.debug(f'GC: utilization={utilization} gc={results} before={before} after={after} device={torch.device(get_optimal_device_name())} fn={fn} time={round(t1 - t0, 2)}') + log.debug(f'GC: current={after} prev={before} load={utilization} gc={results} fn={fn} why={reason} time={t1-t0:.2f}') return new_gpu, new_ram diff --git a/modules/ipadapter.py b/modules/ipadapter.py index 4e93a6eee..c1b6ed52f 100644 --- a/modules/ipadapter.py +++ b/modules/ipadapter.py @@ -9,12 +9,15 @@ import time import json from PIL import Image +import diffusers +import transformers from modules import processing, shared, devices, sd_models -clip_repo = "h94/IP-Adapter" clip_loaded = None adapters_loaded = [] +CLIP_ID = "h94/IP-Adapter" +SIGLIP_ID = 'google/siglip-so400m-patch14-384' ADAPTERS_NONE = { 'None': { 'name': 'none', 'repo': 'none', 'subfolder': 'none' }, } @@ -37,11 +40,13 @@ 'Ostris Composition ViT-H SDXL': { 'name': 'ip_plus_composition_sdxl.safetensors', 'repo': 'ostris/ip-composition-adapter', 'subfolder': '' }, } ADAPTERS_SD3 = { - 'InstantX Large': { 'name': 'ip-adapter.bin', 'repo': 'InstantX/SD3.5-Large-IP-Adapter' }, + 'None': { 'name': 'none', 'repo': 'none', 'subfolder': 'none' }, + 'InstantX Large': { 'name': 'none', 'repo': 'InstantX/SD3.5-Large-IP-Adapter', 'subfolder': 'none', 'revision': 'refs/pr/10' }, } ADAPTERS_F1 = { - 'XLabs AI v1': { 'name': 'ip_adapter.safetensors', 'repo': 'XLabs-AI/flux-ip-adapter' }, - 'XLabs AI v2': { 'name': 'ip_adapter.safetensors', 'repo': 'XLabs-AI/flux-ip-adapter-v2' }, + 'None': { 'name': 'none', 'repo': 'none', 'subfolder': 'none' }, + 'XLabs AI v1': { 'name': 'ip_adapter.safetensors', 'repo': 'XLabs-AI/flux-ip-adapter', 'subfolder': 'none' }, + 'XLabs AI v2': { 'name': 'ip_adapter.safetensors', 'repo': 'XLabs-AI/flux-ip-adapter-v2', 'subfolder': 'none' }, } ADAPTERS = { **ADAPTERS_SD15, **ADAPTERS_SDXL, **ADAPTERS_SD3, **ADAPTERS_F1 } ADAPTERS_ALL = { **ADAPTERS_SD15, **ADAPTERS_SDXL, **ADAPTERS_SD3, **ADAPTERS_F1 } @@ -126,6 +131,8 @@ def crop_images(images, crops): shared.log.error(f'IP adapter: failed to crop image: source={len(images[i])} faces={len(cropped)}') except Exception as e: shared.log.error(f'IP adapter: failed to crop image: {e}') + if shared.sd_model_type == 'sd3' and len(images) == 1: + return images[0] return images @@ -144,27 +151,64 @@ def unapply(pipe): # pylint: disable=arguments-differ pass -def apply(pipe, p: processing.StableDiffusionProcessing, adapter_names=[], adapter_scales=[1.0], adapter_crops=[False], adapter_starts=[0.0], adapter_ends=[1.0], adapter_images=[]): - global clip_loaded, adapters_loaded # pylint: disable=global-statement - # overrides - if hasattr(p, 'ip_adapter_names'): - if isinstance(p.ip_adapter_names, str): - p.ip_adapter_names = [p.ip_adapter_names] - adapters = [ADAPTERS_ALL.get(adapter_name, None) for adapter_name in p.ip_adapter_names if adapter_name is not None and adapter_name.lower() != 'none'] - adapter_names = p.ip_adapter_names - else: - if isinstance(adapter_names, str): - adapter_names = [adapter_names] - adapters = [ADAPTERS.get(adapter, None) for adapter in adapter_names] - adapters = [adapter for adapter in adapters if adapter is not None and adapter['name'].lower() != 'none'] - if len(adapters) == 0: - unapply(pipe) - if hasattr(p, 'ip_adapter_images'): - del p.ip_adapter_images - return False - if shared.sd_model_type not in ['sd', 'sdxl', 'sd3', 'f1']: - shared.log.error(f'IP adapter: model={shared.sd_model_type} class={pipe.__class__.__name__} not supported') - return False +def load_image_encoder(pipe: diffusers.DiffusionPipeline, adapter_names: list[str]): + global clip_loaded # pylint: disable=global-statement + for adapter_name in adapter_names: + # which clip to use + clip_repo = CLIP_ID + if 'ViT' not in adapter_name: # defaults per model + clip_subfolder = 'models/image_encoder' if shared.sd_model_type == 'sd' else 'sdxl_models/image_encoder' + if 'ViT-H' in adapter_name: + clip_subfolder = 'models/image_encoder' # this is vit-h + elif 'ViT-G' in adapter_name: + clip_subfolder = 'sdxl_models/image_encoder' # this is vit-g + else: + if shared.sd_model_type == 'sd': + clip_subfolder = 'models/image_encoder' + elif shared.sd_model_type == 'sdxl': + clip_subfolder = 'sdxl_models/image_encoder' + elif shared.sd_model_type == 'sd3': + clip_repo = SIGLIP_ID + clip_subfolder = None + elif shared.sd_model_type == 'f1': + shared.log.error(f'IP adapter: adapter={adapter_name} type={shared.sd_model_type} cls={shared.sd_model.__class__.__name__}: unsupported base model') + return False + else: + shared.log.error(f'IP adapter: unknown model type: {adapter_name}') + return False + + # load image encoder used by ip adapter + if pipe.image_encoder is None or clip_loaded != f'{clip_repo}/{clip_subfolder}': + try: + if shared.sd_model_type == 'sd3': + pipe.image_encoder = transformers.SiglipVisionModel.from_pretrained(clip_repo, torch_dtype=devices.dtype, cache_dir=shared.opts.hfcache_dir) + else: + pipe.image_encoder = transformers.CLIPVisionModelWithProjection.from_pretrained(clip_repo, subfolder=clip_subfolder, torch_dtype=devices.dtype, cache_dir=shared.opts.hfcache_dir, use_safetensors=True) + shared.log.debug(f'IP adapter load: encoder="{clip_repo}/{clip_subfolder}" cls={pipe.image_encoder.__class__.__name__}') + clip_loaded = f'{clip_repo}/{clip_subfolder}' + except Exception as e: + shared.log.error(f'IP adapter load: encoder="{clip_repo}/{clip_subfolder}" {e}') + return False + sd_models.move_model(pipe.image_encoder, devices.device) + return True + + +def load_feature_extractor(pipe): + # load feature extractor used by ip adapter + if pipe.feature_extractor is None: + try: + if shared.sd_model_type == 'sd3': + pipe.feature_extractor = transformers.SiglipImageProcessor.from_pretrained(SIGLIP_ID, torch_dtype=devices.dtype, cache_dir=shared.opts.hfcache_dir) + else: + pipe.feature_extractor = transformers.CLIPImageProcessor() + shared.log.debug(f'IP adapter load: extractor={pipe.feature_extractor.__class__.__name__}') + except Exception as e: + shared.log.error(f'IP adapter load: extractor {e}') + return False + return True + + +def parse_params(p: processing.StableDiffusionProcessing, adapters: list, adapter_scales: list[float], adapter_crops: list[bool], adapter_starts: list[float], adapter_ends: list[float], adapter_images: list): if hasattr(p, 'ip_adapter_scales'): adapter_scales = p.ip_adapter_scales if hasattr(p, 'ip_adapter_crops'): @@ -205,6 +249,33 @@ def apply(pipe, p: processing.StableDiffusionProcessing, adapter_names=[], adapt p.ip_adapter_starts = adapter_starts.copy() adapter_ends = get_scales(adapter_ends, adapter_images) p.ip_adapter_ends = adapter_ends.copy() + return adapter_images, adapter_masks, adapter_scales, adapter_crops, adapter_starts, adapter_ends + + +def apply(pipe, p: processing.StableDiffusionProcessing, adapter_names=[], adapter_scales=[1.0], adapter_crops=[False], adapter_starts=[0.0], adapter_ends=[1.0], adapter_images=[]): + global adapters_loaded # pylint: disable=global-statement + # overrides + if hasattr(p, 'ip_adapter_names'): + if isinstance(p.ip_adapter_names, str): + p.ip_adapter_names = [p.ip_adapter_names] + adapters = [ADAPTERS_ALL.get(adapter_name, None) for adapter_name in p.ip_adapter_names if adapter_name is not None and adapter_name.lower() != 'none'] + adapter_names = p.ip_adapter_names + else: + if isinstance(adapter_names, str): + adapter_names = [adapter_names] + adapters = [ADAPTERS.get(adapter_name, None) for adapter_name in adapter_names if adapter_name.lower() != 'none'] + + if len(adapters) == 0: + unapply(pipe) + if hasattr(p, 'ip_adapter_images'): + del p.ip_adapter_images + return False + if shared.sd_model_type not in ['sd', 'sdxl', 'sd3', 'f1']: + shared.log.error(f'IP adapter: model={shared.sd_model_type} class={pipe.__class__.__name__} not supported') + return False + + adapter_images, adapter_masks, adapter_scales, adapter_crops, adapter_starts, adapter_ends = parse_params(p, adapters, adapter_scales, adapter_crops, adapter_starts, adapter_ends, adapter_images) + # init code if pipe is None: return False @@ -223,61 +294,29 @@ def apply(pipe, p: processing.StableDiffusionProcessing, adapter_names=[], adapt shared.log.error(f'IP adapter: pipeline not supported: {pipe.__class__.__name__}') return False - for adapter_name in adapter_names: - # which clip to use - if 'ViT' not in adapter_name: # defaults per model - if shared.sd_model_type == 'sd': - clip_subfolder = 'models/image_encoder' - else: - clip_subfolder = 'sdxl_models/image_encoder' - if 'ViT-H' in adapter_name: - clip_subfolder = 'models/image_encoder' # this is vit-h - elif 'ViT-G' in adapter_name: - clip_subfolder = 'sdxl_models/image_encoder' # this is vit-g - else: - if shared.sd_model_type == 'sd': - clip_subfolder = 'models/image_encoder' - elif shared.sd_model_type == 'sdxl': - clip_subfolder = 'sdxl_models/image_encoder' - elif shared.sd_model_type == 'sd3': - shared.log.error(f'IP adapter: adapter={adapter_name} type={shared.sd_model_type} cls={shared.sd_model.__class__.__name__}: unsupported base model') - return False - elif shared.sd_model_type == 'f1': - shared.log.error(f'IP adapter: adapter={adapter_name} type={shared.sd_model_type} cls={shared.sd_model.__class__.__name__}: unsupported base model') - return False - else: - shared.log.error(f'IP adapter: unknown model type: {adapter_name}') - return False - - # load feature extractor used by ip adapter - if pipe.feature_extractor is None: - try: - from transformers import CLIPImageProcessor - shared.log.debug('IP adapter load: feature extractor') - pipe.feature_extractor = CLIPImageProcessor() - except Exception as e: - shared.log.error(f'IP adapter load: feature extractor {e}') - return False + if not load_image_encoder(pipe, adapter_names): + return False - # load image encoder used by ip adapter - if pipe.image_encoder is None or clip_loaded != f'{clip_repo}/{clip_subfolder}': - try: - from transformers import CLIPVisionModelWithProjection - shared.log.debug(f'IP adapter load: image encoder="{clip_repo}/{clip_subfolder}"') - pipe.image_encoder = CLIPVisionModelWithProjection.from_pretrained(clip_repo, subfolder=clip_subfolder, torch_dtype=devices.dtype, cache_dir=shared.opts.diffusers_dir, use_safetensors=True) - clip_loaded = f'{clip_repo}/{clip_subfolder}' - except Exception as e: - shared.log.error(f'IP adapter load: image encoder="{clip_repo}/{clip_subfolder}" {e}') - return False - sd_models.move_model(pipe.image_encoder, devices.device) + if not load_feature_extractor(pipe): + return False # main code try: t0 = time.time() - repos = [adapter['repo'] for adapter in adapters] - subfolders = [adapter['subfolder'] for adapter in adapters] - names = [adapter['name'] for adapter in adapters] - pipe.load_ip_adapter(repos, subfolder=subfolders, weight_name=names) + repos = [adapter.get('repo', None) for adapter in adapters if adapter.get('repo', 'none') != 'none'] + subfolders = [adapter.get('subfolder', None) for adapter in adapters if adapter.get('subfolder', 'none') != 'none'] + names = [adapter.get('name', None) for adapter in adapters if adapter.get('name', 'none') != 'none'] + revisions = [adapter.get('revision', None) for adapter in adapters if adapter.get('revision', 'none') != 'none'] + kwargs = {} + if len(repos) == 1: + repos = repos[0] + if len(subfolders) > 0: + kwargs['subfolder'] = subfolders if len(subfolders) > 1 else subfolders[0] + if len(names) > 0: + kwargs['weight_name'] = names if len(names) > 1 else names[0] + if len(revisions) > 0: + kwargs['revision'] = revisions[0] + pipe.load_ip_adapter(repos, **kwargs) adapters_loaded = names if hasattr(p, 'ip_adapter_layers'): pipe.set_ip_adapter_scale(p.ip_adapter_layers) @@ -286,8 +325,8 @@ def apply(pipe, p: processing.StableDiffusionProcessing, adapter_names=[], adapt for i in range(len(adapter_scales)): if adapter_starts[i] > 0: adapter_scales[i] = 0.00 - pipe.set_ip_adapter_scale(adapter_scales) - ip_str = [f'{os.path.splitext(adapter)[0]}:{scale}:{start}:{end}' for adapter, scale, start, end in zip(adapter_names, adapter_scales, adapter_starts, adapter_ends)] + pipe.set_ip_adapter_scale(adapter_scales if len(adapter_scales) > 1 else adapter_scales[0]) + ip_str = [f'{os.path.splitext(adapter)[0]}:{scale}:{start}:{end}:{crop}' for adapter, scale, start, end, crop in zip(adapter_names, adapter_scales, adapter_starts, adapter_ends, adapter_crops)] p.task_args['ip_adapter_image'] = crop_images(adapter_images, adapter_crops) if len(adapter_masks) > 0: p.cross_attention_kwargs = { 'ip_adapter_masks': adapter_masks } diff --git a/modules/processing.py b/modules/processing.py index 4bc5e3c81..f39393f95 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -480,5 +480,5 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: if not p.disable_extra_networks: shared.log.info(f'Processed: images={len(output_images)} its={(p.steps * len(output_images)) / (t1 - t0):.2f} time={t1-t0:.2f} timers={timer.process.dct()} memory={memstats.memory_stats()}') - devices.torch_gc(force=True) + devices.torch_gc(force=True, reason='final') return processed diff --git a/modules/sd_models.py b/modules/sd_models.py index 52d0a1deb..a3698c5c7 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -496,7 +496,11 @@ def apply_balanced_offload_to_module(pipe): used_gpu -= module_size debug_move(f'Offload: type=balanced op={"move" if do_offload else "skip"} gpu={prev_gpu:.3f}:{used_gpu:.3f} perc={perc_gpu:.2f} ram={used_ram:.3f} current={module.device} dtype={module.dtype} component={module.__class__.__name__} size={module_size:.3f}') except Exception as e: - if 'bitsandbytes' not in str(e): + if 'out of memory' in str(e): + devices.torch_gc(fast=True, force=True, reason='oom') + elif 'bitsandbytes' in str(e): + pass + else: shared.log.error(f'Offload: type=balanced op=apply module={module_name} {e}') if os.environ.get('SD_MOVE_DEBUG', None): errors.display(e, f'Offload: type=balanced op=apply module={module_name}') @@ -508,7 +512,7 @@ def apply_balanced_offload_to_module(pipe): if device_map and max_memory: module.balanced_offload_device_map = device_map module.balanced_offload_max_memory = max_memory - devices.torch_gc(fast=True, force=True) + devices.torch_gc(fast=True, force=True, reason='offload') apply_balanced_offload_to_module(sd_model) if hasattr(sd_model, "pipe"): @@ -518,7 +522,6 @@ def apply_balanced_offload_to_module(pipe): if hasattr(sd_model, "decoder_pipe"): apply_balanced_offload_to_module(sd_model.decoder_pipe) set_accelerate(sd_model) - devices.torch_gc(fast=True) t = time.time() - t0 process_timer.add('offload', t) fn = f'{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access diff --git a/scripts/flux_tools.py b/scripts/flux_tools.py index 909257a37..50904eedb 100644 --- a/scripts/flux_tools.py +++ b/scripts/flux_tools.py @@ -7,7 +7,8 @@ from installer import install -redux_pipe: diffusers.FluxPriorReduxPipeline = None +# redux_pipe: diffusers.FluxPriorReduxPipeline = None +redux_pipe = None processor_canny = None processor_depth = None title = 'Flux Tools' From 14e4a5a9664a325b41e395e8f442784f1e2dbbd6 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 20 Dec 2024 10:37:11 -0500 Subject: [PATCH 117/249] notify on torch install Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 1 + installer.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d2d70f3e1..9398e533b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -200,6 +200,7 @@ And it wouldn't be a Xmass edition without couple of custom themes: *Snowflake* - taesd downscale preview to 1024px max: configurable in settings -> live preview - uninstall conflicting `wandb` package - dont skip diffusers version check if quick is specified +- notify on torch install ## Update for 2024-11-21 diff --git a/installer.py b/installer.py index f1360541f..40ba7cea0 100644 --- a/installer.py +++ b/installer.py @@ -751,6 +751,8 @@ def check_torch(): log.warning('Torch: CPU-only version installed') torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision') if 'torch' in torch_command and not args.version: + if not installed('torch'): + log.info(f'Torch: download and install in progress... cmd="{torch_command}"') install(torch_command, 'torch torchvision', quiet=True) else: try: From ff93e1b9fb4ce14c19393bf609d08babd494896f Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sat, 21 Dec 2024 01:25:49 +0300 Subject: [PATCH 118/249] IPEX dupe conv2d fix for conv1d and conv3d too --- modules/intel/ipex/hijacks.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/modules/intel/ipex/hijacks.py b/modules/intel/ipex/hijacks.py index 7ec94138d..43721a64c 100644 --- a/modules/intel/ipex/hijacks.py +++ b/modules/intel/ipex/hijacks.py @@ -149,6 +149,15 @@ def functional_linear(input, weight, bias=None): bias.data = bias.data.to(dtype=weight.data.dtype) return original_functional_linear(input, weight, bias=bias) +original_functional_conv1d = torch.nn.functional.conv1d +@wraps(torch.nn.functional.conv1d) +def functional_conv1d(input, weight, bias=None, stride=1, padding=0, dilation=1, groups=1): + if input.dtype != weight.data.dtype: + input = input.to(dtype=weight.data.dtype) + if bias is not None and bias.data.dtype != weight.data.dtype: + bias.data = bias.data.to(dtype=weight.data.dtype) + return original_functional_conv1d(input, weight, bias=bias, stride=stride, padding=padding, dilation=dilation, groups=groups) + original_functional_conv2d = torch.nn.functional.conv2d @wraps(torch.nn.functional.conv2d) def functional_conv2d(input, weight, bias=None, stride=1, padding=0, dilation=1, groups=1): @@ -158,6 +167,16 @@ def functional_conv2d(input, weight, bias=None, stride=1, padding=0, dilation=1, bias.data = bias.data.to(dtype=weight.data.dtype) return original_functional_conv2d(input, weight, bias=bias, stride=stride, padding=padding, dilation=dilation, groups=groups) +# LTX Video +original_functional_conv3d = torch.nn.functional.conv3d +@wraps(torch.nn.functional.conv3d) +def functional_conv3d(input, weight, bias=None, stride=1, padding=0, dilation=1, groups=1): + if input.dtype != weight.data.dtype: + input = input.to(dtype=weight.data.dtype) + if bias is not None and bias.data.dtype != weight.data.dtype: + bias.data = bias.data.to(dtype=weight.data.dtype) + return original_functional_conv3d(input, weight, bias=bias, stride=stride, padding=padding, dilation=dilation, groups=groups) + # SwinIR BF16: original_functional_pad = torch.nn.functional.pad @wraps(torch.nn.functional.pad) @@ -320,7 +339,9 @@ def ipex_hijacks(legacy=True): torch.nn.functional.group_norm = functional_group_norm torch.nn.functional.layer_norm = functional_layer_norm torch.nn.functional.linear = functional_linear + torch.nn.functional.conv1d = functional_conv1d torch.nn.functional.conv2d = functional_conv2d + torch.nn.functional.conv3d = functional_conv3d torch.nn.functional.pad = functional_pad torch.bmm = torch_bmm From ee594e6b9dfe2c84328729633719758da69fc57f Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 20 Dec 2024 17:28:32 -0500 Subject: [PATCH 119/249] on-the-fly quant for sd35, flux and sana Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 16 ++--- html/reference.json | 12 +++- modules/model_flux.py | 33 +++++++-- modules/model_sana.py | 67 ++++++++++++------- modules/model_sd3.py | 20 +++--- modules/sd_models.py | 18 +++-- .../textual_inversion/textual_inversion.py | 5 +- wiki | 2 +- 8 files changed, 113 insertions(+), 60 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9398e533b..ebf17d6b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,13 +1,13 @@ # Change Log for SD.Next -## Update for 2024-12-19 +## Update for 2024-12-20 -### Highlights for 2024-12-19 +### Highlights for 2024-12-20 ### SD.Next Xmass edition: *What's new?* While we have several new supported models, workflows and tools, this release is primarily about *quality-of-life improvements*: -- New memory management engine: list of changes that went into this one is long: changes to GPU offloading, LoRA loader, system memory management, etc. +- New memory management engine: list of changes that went into this one is long: changes to GPU offloading, LoRA loader, system memory management, on-the-fly quantization, etc. but main goal is enabling modern large models to run on standard consumer GPUs without performance hits typically associated with aggressive memory swapping and needs for constant manual tweaks - New [documentation website](https://vladmandic.github.io/sdnext-docs/) @@ -17,28 +17,28 @@ While we have several new supported models, workflows and tools, this release is We've also added support for several new models (see [supported models](https://vladmandic.github.io/sdnext-docs/Model-Support/) for full list) such as highly anticipated [NVLabs Sana](https://huggingface.co/Efficient-Large-Model/Sana_1600M_1024px) And several new SOTA video models: [Lightricks LTX-Video](https://huggingface.co/Lightricks/LTX-Video), [Hunyuan Video](https://huggingface.co/tencent/HunyuanVideo) and [Genmo Mochi.1 Preview](https://huggingface.co/genmo/mochi-1-preview) -And a lot of Control and IPAdapter goodies +And a lot of **Control** and **IPAdapter** goodies - for **SDXL** there is new [ProMax](https://huggingface.co/xinsir/controlnet-union-sdxl-1.0), improved *Union* and *Tiling* models - for **FLUX.1** there are [Flux Tools](https://blackforestlabs.ai/flux-1-tools/) as well as official *Canny* and *Depth* models and a cool [Redux](https://huggingface.co/black-forest-labs/FLUX.1-Redux-dev) model - for **SD3.5** there are official *Canny*, *Blur* and *Depth* models in addition to existing 3rd party models as well as [InstantX](https://huggingface.co/InstantX/SD3.5-Large-IP-Adapter) IP-adapter Plus couple of new integrated workflows such as [FreeScale](https://github.com/ali-vilab/FreeScale) and [Style Aligned Image Generation](https://style-aligned-gen.github.io/) -And it wouldn't be a Xmass edition without couple of custom themes: *Snowflake* and *Elf-Green*! +And it wouldn't be a *Xmass edition* without couple of custom themes: *Snowflake* and *Elf-Green*! [ReadMe](https://github.com/vladmandic/automatic/blob/master/README.md) | [ChangeLog](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) | [Docs](https://vladmandic.github.io/sdnext-docs/) | [WiKi](https://github.com/vladmandic/automatic/wiki) | [Discord](https://discord.com/invite/sd-next-federal-batch-inspectors-1101998836328697867) -## Details for 2024-12-19 +## Details for 2024-12-20 ### New models and integrations - [NVLabs Sana](https://huggingface.co/Efficient-Large-Model/Sana_1600M_1024px) - support for both 1.6B and 0.6B models + support for 1.6B 2048px, 1.6B 1024px and 0.6B 512px models **Sana** can synthesize high-resolution images with strong text-image alignment by using **Gemma2** as text-encoder and its *fast* - typically at least **2x** faster than sd-xl even for 1.6B variant and maintains performance regardless of resolution e.g., rendering at 4k is possible in less than 8GB vram to use, select from *networks -> models -> reference* and models will be auto-downloaded on first use - *reference values*: sampler: default (or any flow-match variant), width/height: 1024, guidance scale: 4.5 + *reference values*: sampler: default (or any flow-match variant), steps: 20, width/height: 1024, guidance scale: 4.5 *note* like other LLM-based text-encoders, sana prefers long and descriptive prompts any short prompt below 300 characters will be auto-expanded using built in Gemma LLM before encoding while long prompts will be passed as-is - **ControlNet** diff --git a/html/reference.json b/html/reference.json index 8a0965697..43115c549 100644 --- a/html/reference.json +++ b/html/reference.json @@ -180,14 +180,20 @@ "extras": "sampler: Default, cfg_scale: 3.5" }, - "NVLabs Sana 1.6B": { + "NVLabs Sana 1.6B 2048px": { + "path": "Efficient-Large-Model/Sana_1600M_2Kpx_BF16_diffusers", + "desc": "Sana is a text-to-image framework that can efficiently generate images up to 4096 × 4096 resolution. Sana can synthesize high-resolution, high-quality images with strong text-image alignment at a remarkably fast speed, deployable on laptop GPU.", + "preview": "Efficient-Large-Model--Sana_1600M_1024px_diffusers.jpg", + "skip": true + }, + "NVLabs Sana 1.6B 1024px": { "path": "Efficient-Large-Model/Sana_1600M_1024px_diffusers", "desc": "Sana is a text-to-image framework that can efficiently generate images up to 4096 × 4096 resolution. Sana can synthesize high-resolution, high-quality images with strong text-image alignment at a remarkably fast speed, deployable on laptop GPU.", "preview": "Efficient-Large-Model--Sana_1600M_1024px_diffusers.jpg", "skip": true }, - "NVLabs Sana 0.6B": { - "path": "Efficient-Large-Model/Sana_600M_1024px_diffusers", + "NVLabs Sana 0.6B 512px": { + "path": "Efficient-Large-Model/Sana_600M_512px_diffusers", "desc": "Sana is a text-to-image framework that can efficiently generate images up to 4096 × 4096 resolution. Sana can synthesize high-resolution, high-quality images with strong text-image alignment at a remarkably fast speed, deployable on laptop GPU.", "preview": "Efficient-Large-Model--Sana_1600M_1024px_diffusers.jpg", "skip": true diff --git a/modules/model_flux.py b/modules/model_flux.py index 362f96d9b..ac1370ef7 100644 --- a/modules/model_flux.py +++ b/modules/model_flux.py @@ -110,6 +110,7 @@ def load_flux_bnb(checkpoint_info, diffusers_load_config): # pylint: disable=unu return transformer, text_encoder_2 +""" def quant_flux_bnb(checkpoint_info, transformer, text_encoder_2): repo_id = sd_models.path_to_repo(checkpoint_info.name) cache_dir=shared.opts.diffusers_dir @@ -139,6 +140,24 @@ def quant_flux_bnb(checkpoint_info, transformer, text_encoder_2): from modules import errors errors.display(e, 'FLUX:') return transformer, text_encoder_2 +""" + + +def load_quants(kwargs, repo_id, cache_dir): + if len(shared.opts.bnb_quantization) > 0: + quant_args = {} + quant_args = model_quant.create_bnb_config(quant_args) + quant_args = model_quant.create_ao_config(quant_args) + if not quant_args: + return + model_quant.load_bnb(f'Load model: type=FLUX quant={quant_args}') + if 'Model' in shared.opts.bnb_quantization and 'transformer' not in kwargs: + kwargs['transformer'] = diffusers.FluxTransformer2DModel.from_pretrained(repo_id, subfolder="transformer", cache_dir=cache_dir, torch_dtype=devices.dtype, **quant_args) + shared.log.debug(f'Quantization: module=transformer type=bnb dtype={shared.opts.bnb_quantization_type} storage={shared.opts.bnb_quantization_storage}') + if 'Text Encoder' in shared.opts.bnb_quantization and 'text_encoder_3' not in kwargs: + kwargs['text_encoder_2'] = transformers.T5EncoderModel.from_pretrained(repo_id, subfolder="text_encoder_2", cache_dir=cache_dir, torch_dtype=devices.dtype, **quant_args) + shared.log.debug(f'Quantization: module=t5 type=bnb dtype={shared.opts.bnb_quantization_type} storage={shared.opts.bnb_quantization_storage}') + return kwargs def load_flux_gguf(file_path): @@ -148,9 +167,8 @@ def load_flux_gguf(file_path): from diffusers.loaders.single_file_utils import convert_flux_transformer_checkpoint_to_diffusers from modules import ggml, sd_hijack_accelerate with init_empty_weights(): - from diffusers import FluxTransformer2DModel - config = FluxTransformer2DModel.load_config(os.path.join('configs', 'flux'), subfolder="transformer") - transformer = FluxTransformer2DModel.from_config(config).to(devices.dtype) + config = diffusers.FluxTransformer2DModel.load_config(os.path.join('configs', 'flux'), subfolder="transformer") + transformer = diffusers.FluxTransformer2DModel.from_config(config).to(devices.dtype) expected_state_dict_keys = list(transformer.state_dict().keys()) state_dict, stats = ggml.load_gguf_state_dict(file_path, devices.dtype) state_dict = convert_flux_transformer_checkpoint_to_diffusers(state_dict) @@ -295,7 +313,6 @@ def load_flux(checkpoint_info, diffusers_load_config): # triggered by opts.sd_ch # initialize pipeline with pre-loaded components kwargs = {} - # transformer, text_encoder_2 = quant_flux_bnb(checkpoint_info, transformer, text_encoder_2) if transformer is not None: kwargs['transformer'] = transformer sd_unet.loaded_unet = shared.opts.sd_unet @@ -324,10 +341,14 @@ def load_flux(checkpoint_info, diffusers_load_config): # triggered by opts.sd_ch kwargs[c] = kwargs[c].to(dtype=devices.dtype) allow_quant = 'gguf' not in (sd_unet.loaded_unet or '') + fn = checkpoint_info.path + if (fn is None) or (not os.path.exists(fn) or os.path.isdir(fn)): + # transformer, text_encoder_2 = quant_flux_bnb(checkpoint_info, transformer, text_encoder_2) + kwargs = load_quants(kwargs, repo_id, cache_dir=shared.opts.diffusers_dir) kwargs = model_quant.create_bnb_config(kwargs, allow_quant) kwargs = model_quant.create_ao_config(kwargs, allow_quant) - if checkpoint_info.path.endswith('.safetensors') and os.path.isfile(checkpoint_info.path): - pipe = diffusers.FluxPipeline.from_single_file(checkpoint_info.path, cache_dir=shared.opts.diffusers_dir, **kwargs, **diffusers_load_config) + if fn.endswith('.safetensors') and os.path.isfile(fn): + pipe = diffusers.FluxPipeline.from_single_file(fn, cache_dir=shared.opts.diffusers_dir, **kwargs, **diffusers_load_config) else: pipe = cls.from_pretrained(repo_id, cache_dir=shared.opts.diffusers_dir, **kwargs, **diffusers_load_config) diff --git a/modules/model_sana.py b/modules/model_sana.py index c25a7ffb9..414f9b74d 100644 --- a/modules/model_sana.py +++ b/modules/model_sana.py @@ -1,54 +1,73 @@ +import os import time import torch import diffusers +import transformers +from modules import shared, sd_models, devices, modelloader, model_quant -""" -Efficient-Large-Model/Sana_1600M_1024px_MultiLing_diffusers -Efficient-Large-Model/Sana_1600M_1024px_diffusers -Efficient-Large-Model/Sana_1600M_1024px_BF16_diffusers -Efficient-Large-Model/Sana_1600M_512px_MultiLing_diffusers -Efficient-Large-Model/Sana_1600M_512px_diffusers -Efficient-Large-Model/Sana_600M_1024px_diffusers -Efficient-Large-Model/Sana_600M_512px_diffusers -""" +def load_quants(kwargs, repo_id, cache_dir): + if len(shared.opts.bnb_quantization) > 0: + quant_args = {} + quant_args = model_quant.create_bnb_config(quant_args) + quant_args = model_quant.create_ao_config(quant_args) + load_args = kwargs.copy() + if not quant_args: + return + model_quant.load_bnb(f'Load model: type=SD3 quant={quant_args} args={load_args}') + if 'Model' in shared.opts.bnb_quantization and 'transformer' not in kwargs: + kwargs['transformer'] = diffusers.models.SanaTransformer2DModel.from_pretrained(repo_id, subfolder="transformer", cache_dir=cache_dir, **load_args, **quant_args) + shared.log.debug(f'Quantization: module=transformer type=bnb dtype={shared.opts.bnb_quantization_type} storage={shared.opts.bnb_quantization_storage}') + if 'Text Encoder' in shared.opts.bnb_quantization and 'text_encoder_3' not in kwargs: + kwargs['text_encoder_3'] = transformers.AutoModelForCausalLM.from_pretrained(repo_id, subfolder="text_encoder", cache_dir=cache_dir, **load_args, **quant_args) + shared.log.debug(f'Quantization: module=t5 type=bnb dtype={shared.opts.bnb_quantization_type} storage={shared.opts.bnb_quantization_storage}') + return kwargs def load_sana(checkpoint_info, kwargs={}): - from modules import shared, sd_models, devices, modelloader, model_quant modelloader.hf_login() - repo_id = checkpoint_info if isinstance(checkpoint_info, str) else checkpoint_info.path - repo_id = sd_models.path_to_repo(repo_id) + fn = checkpoint_info if isinstance(checkpoint_info, str) else checkpoint_info.path + repo_id = sd_models.path_to_repo(fn) kwargs.pop('load_connected_pipeline', None) kwargs.pop('safety_checker', None) kwargs.pop('requires_safety_checker', None) kwargs.pop('torch_dtype', None) + if not repo_id.endswith('_diffusers'): + repo_id = f'{repo_id}_diffusers' + if devices.dtype == torch.bfloat16 and 'BF16' not in repo_id: + repo_id = repo_id.replace('_diffusers', '_BF16_diffusers') + if 'Sana_1600M' in repo_id: - if devices.dtype == torch.bfloat16: - repo_id = 'Efficient-Large-Model/Sana_1600M_1024px_BF16_diffusers' + if devices.dtype == torch.bfloat16 or 'BF16' in repo_id: + if 'BF16' not in repo_id: + repo_id = repo_id.replace('_diffusers', '_BF16_diffusers') kwargs['variant'] = 'bf16' kwargs['torch_dtype'] = devices.dtype else: - repo_id = 'Efficient-Large-Model/Sana_1600M_1024px_diffusers' kwargs['variant'] = 'fp16' if 'Sana_600M' in repo_id: - repo_id = 'Efficient-Large-Model/Sana_600M_1024px_diffusers' kwargs['variant'] = 'fp16' - kwargs = model_quant.create_bnb_config(kwargs) - kwargs = model_quant.create_ao_config(kwargs) - shared.log.debug(f'Load model: type=Sana repo="{repo_id}" args={kwargs}') + if (fn is None) or (not os.path.exists(fn) or os.path.isdir(fn)): + kwargs = load_quants(kwargs, repo_id, cache_dir=shared.opts.diffusers_dir) + # kwargs = model_quant.create_bnb_config(kwargs) + # kwargs = model_quant.create_ao_config(kwargs) + shared.log.debug(f'Load model: type=Sana repo="{repo_id}" args={list(kwargs)}') t0 = time.time() - pipe = diffusers.SanaPipeline.from_pretrained(repo_id, cache_dir = shared.opts.diffusers_dir, **kwargs) + pipe = diffusers.SanaPipeline.from_pretrained(repo_id, cache_dir=shared.opts.diffusers_dir, **kwargs) if devices.dtype == torch.bfloat16 or devices.dtype == torch.float32: - pipe.transformer = pipe.transformer.to(dtype=devices.dtype) - pipe.text_encoder = pipe.text_encoder.to(dtype=devices.dtype) + if 'transformer' not in kwargs: + pipe.transformer = pipe.transformer.to(dtype=devices.dtype) + if 'text_encoder' not in kwargs: + pipe.text_encoder = pipe.text_encoder.to(dtype=devices.dtype) pipe.vae = pipe.vae.to(dtype=devices.dtype) if devices.dtype == torch.float16: - pipe.transformer = pipe.transformer.to(dtype=devices.dtype) - pipe.text_encoder = pipe.text_encoder.to(dtype=torch.float32) # gemma2 does not support fp16 + if 'transformer' not in kwargs: + pipe.transformer = pipe.transformer.to(dtype=devices.dtype) + if 'text_encoder' not in kwargs: + pipe.text_encoder = pipe.text_encoder.to(dtype=torch.float32) # gemma2 does not support fp16 pipe.vae = pipe.vae.to(dtype=torch.float32) # dc-ae often overflows in fp16 if shared.opts.diffusers_eval: pipe.text_encoder.eval() diff --git a/modules/model_sd3.py b/modules/model_sd3.py index ba036760a..2842661bd 100644 --- a/modules/model_sd3.py +++ b/modules/model_sd3.py @@ -51,19 +51,17 @@ def load_overrides(kwargs, cache_dir): def load_quants(kwargs, repo_id, cache_dir): if len(shared.opts.bnb_quantization) > 0: - model_quant.load_bnb('Load model: type=SD3') - bnb_config = diffusers.BitsAndBytesConfig( - load_in_8bit=shared.opts.bnb_quantization_type in ['fp8'], - load_in_4bit=shared.opts.bnb_quantization_type in ['nf4', 'fp4'], - bnb_4bit_quant_storage=shared.opts.bnb_quantization_storage, - bnb_4bit_quant_type=shared.opts.bnb_quantization_type, - bnb_4bit_compute_dtype=devices.dtype - ) + quant_args = {} + quant_args = model_quant.create_bnb_config(quant_args) + quant_args = model_quant.create_ao_config(quant_args) + if not quant_args: + return + model_quant.load_bnb(f'Load model: type=SD3 quant={quant_args}') if 'Model' in shared.opts.bnb_quantization and 'transformer' not in kwargs: - kwargs['transformer'] = diffusers.SD3Transformer2DModel.from_pretrained(repo_id, subfolder="transformer", cache_dir=cache_dir, quantization_config=bnb_config, torch_dtype=devices.dtype) + kwargs['transformer'] = diffusers.SD3Transformer2DModel.from_pretrained(repo_id, subfolder="transformer", cache_dir=cache_dir, torch_dtype=devices.dtype, **quant_args) shared.log.debug(f'Quantization: module=transformer type=bnb dtype={shared.opts.bnb_quantization_type} storage={shared.opts.bnb_quantization_storage}') if 'Text Encoder' in shared.opts.bnb_quantization and 'text_encoder_3' not in kwargs: - kwargs['text_encoder_3'] = transformers.T5EncoderModel.from_pretrained(repo_id, subfolder="text_encoder_3", variant='fp16', cache_dir=cache_dir, quantization_config=bnb_config, torch_dtype=devices.dtype) + kwargs['text_encoder_3'] = transformers.T5EncoderModel.from_pretrained(repo_id, subfolder="text_encoder_3", variant='fp16', cache_dir=cache_dir, torch_dtype=devices.dtype, **quant_args) shared.log.debug(f'Quantization: module=t5 type=bnb dtype={shared.opts.bnb_quantization_type} storage={shared.opts.bnb_quantization_storage}') return kwargs @@ -127,7 +125,7 @@ def load_sd3(checkpoint_info, cache_dir=None, config=None): kwargs = {} kwargs = load_overrides(kwargs, cache_dir) - if fn is None or not os.path.exists(fn): + if (fn is None) or (not os.path.exists(fn) or os.path.isdir(fn)): kwargs = load_quants(kwargs, repo_id, cache_dir) loader = diffusers.StableDiffusion3Pipeline.from_pretrained diff --git a/modules/sd_models.py b/modules/sd_models.py index a3698c5c7..5939bccbd 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -371,17 +371,19 @@ def set_diffuser_offload(sd_model, op: str = 'model'): class OffloadHook(accelerate.hooks.ModelHook): - def __init__(self): + def __init__(self, checkpoint_name): if shared.opts.diffusers_offload_max_gpu_memory > 1: shared.opts.diffusers_offload_max_gpu_memory = 0.75 if shared.opts.diffusers_offload_max_cpu_memory > 1: shared.opts.diffusers_offload_max_cpu_memory = 0.75 + self.checkpoint_name = checkpoint_name self.min_watermark = shared.opts.diffusers_offload_min_gpu_memory self.max_watermark = shared.opts.diffusers_offload_max_gpu_memory self.cpu_watermark = shared.opts.diffusers_offload_max_cpu_memory self.gpu = int(shared.gpu_memory * shared.opts.diffusers_offload_max_gpu_memory * 1024*1024*1024) self.cpu = int(shared.cpu_memory * shared.opts.diffusers_offload_max_cpu_memory * 1024*1024*1024) self.offload_map = {} + self.param_map = {} gpu = f'{shared.gpu_memory * shared.opts.diffusers_offload_min_gpu_memory:.3f}-{shared.gpu_memory * shared.opts.diffusers_offload_max_gpu_memory}:{shared.gpu_memory}' shared.log.info(f'Offload: type=balanced op=init watermark={self.min_watermark}-{self.max_watermark} gpu={gpu} cpu={shared.cpu_memory:.3f} limit={shared.opts.cuda_mem_fraction:.2f}') self.validate() @@ -440,12 +442,12 @@ def apply_balanced_offload(sd_model, exclude=[]): if sd_model.__class__.__name__ in excluded: return sd_model cached = True - if offload_hook_instance is None or offload_hook_instance.min_watermark != shared.opts.diffusers_offload_min_gpu_memory or offload_hook_instance.max_watermark != shared.opts.diffusers_offload_max_gpu_memory: - cached = False - offload_hook_instance = OffloadHook() checkpoint_name = sd_model.sd_checkpoint_info.name if getattr(sd_model, "sd_checkpoint_info", None) is not None else None if checkpoint_name is None: checkpoint_name = sd_model.__class__.__name__ + if offload_hook_instance is None or offload_hook_instance.min_watermark != shared.opts.diffusers_offload_min_gpu_memory or offload_hook_instance.max_watermark != shared.opts.diffusers_offload_max_gpu_memory or checkpoint_name != offload_hook_instance.checkpoint_name: + cached = False + offload_hook_instance = OffloadHook(checkpoint_name) def get_pipe_modules(pipe): if hasattr(pipe, "_internal_dict"): @@ -461,11 +463,13 @@ def get_pipe_modules(pipe): if not isinstance(module, torch.nn.Module): continue try: - module_size = sum(p.numel()*p.element_size() for p in module.parameters(recurse=True)) / 1024 / 1024 / 1024 + module_size = sum(p.numel() * p.element_size() for p in module.parameters(recurse=True)) / 1024 / 1024 / 1024 + param_num = sum(p.numel() for p in module.parameters(recurse=True)) / 1024 / 1024 / 1024 except Exception as e: shared.log.error(f'Offload: type=balanced op=calc module={module_name} {e}') module_size = 0 offload_hook_instance.offload_map[module_name] = module_size + offload_hook_instance.param_map[module_name] = param_num modules[module_name] = module_size modules = sorted(modules.items(), key=lambda x: x[1], reverse=True) return modules @@ -494,7 +498,9 @@ def apply_balanced_offload_to_module(pipe): if do_offload: module = module.to(devices.cpu, non_blocking=True) used_gpu -= module_size - debug_move(f'Offload: type=balanced op={"move" if do_offload else "skip"} gpu={prev_gpu:.3f}:{used_gpu:.3f} perc={perc_gpu:.2f} ram={used_ram:.3f} current={module.device} dtype={module.dtype} component={module.__class__.__name__} size={module_size:.3f}') + if not cached: + shared.log.debug(f'Offload: type=balanced module={module_name} cls={module.__class__.__name__} dtype={module.dtype} quant={getattr(module, "quantization_method", None)} params={offload_hook_instance.param_map[module_name]:.3f} size={offload_hook_instance.offload_map[module_name]:.3f}') + debug_move(f'Offload: type=balanced op={"move" if do_offload else "skip"} gpu={prev_gpu:.3f}:{used_gpu:.3f} perc={perc_gpu:.2f} ram={used_ram:.3f} current={module.device} dtype={module.dtype} quant={getattr(module, "quantization_method", None)} module={module.__class__.__name__} size={module_size:.3f}') except Exception as e: if 'out of memory' in str(e): devices.torch_gc(fast=True, force=True, reason='oom') diff --git a/modules/textual_inversion/textual_inversion.py b/modules/textual_inversion/textual_inversion.py index 27bb42116..1c6923285 100644 --- a/modules/textual_inversion/textual_inversion.py +++ b/modules/textual_inversion/textual_inversion.py @@ -11,6 +11,7 @@ debug = shared.log.trace if os.environ.get('SD_TI_DEBUG', None) is not None else lambda *args, **kwargs: None debug('Trace: TEXTUAL INVERSION') +supported_models = ['ldm', 'sd', 'sdxl'] def list_embeddings(*dirs): @@ -370,7 +371,7 @@ def load_from_file(self, path, filename): self.skipped_embeddings[name] = embedding def load_from_dir(self, embdir): - if sd_models.model_data.sd_model is None: + if not shared.sd_loaded: shared.log.info('Skipping embeddings load: model not loaded') return if not os.path.isdir(embdir.path): @@ -390,6 +391,8 @@ def load_from_dir(self, embdir): def load_textual_inversion_embeddings(self, force_reload=False): if not shared.sd_loaded: return + if shared.sd_model_type not in supported_models: + return t0 = time.time() if not force_reload: need_reload = False diff --git a/wiki b/wiki index 8db442124..56ba782f7 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 8db44212407343c1855d8811efb61f6e69bd4caa +Subproject commit 56ba782f744bb8f6928f6c365d6ffc547d339548 From c19bfc3e0b3e4d93e87ba9905e34159d14837d2a Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 20 Dec 2024 18:26:25 -0500 Subject: [PATCH 120/249] lora-direct with bnb Signed-off-by: Vladimir Mandic --- modules/control/run.py | 5 +++-- modules/lora/networks.py | 38 ++++++++++++++++++++++++++------------ modules/scripts.py | 1 + 3 files changed, 30 insertions(+), 14 deletions(-) diff --git a/modules/control/run.py b/modules/control/run.py index e780b9bae..efe9c0b8b 100644 --- a/modules/control/run.py +++ b/modules/control/run.py @@ -698,7 +698,7 @@ def control_run(state: str = '', # actual processing if p.is_tile: processed: processing.Processed = tile.run_tiling(p, input_image) - if processed is None: + if processed is None and p.scripts is not None: processed = p.scripts.run(p, *p.script_args) if processed is None: processed: processing.Processed = processing.process_images(p) # run actual pipeline @@ -706,7 +706,8 @@ def control_run(state: str = '', script_run = True # postprocessing - processed = p.scripts.after(p, processed, *p.script_args) + if p.scripts is not None: + processed = p.scripts.after(p, processed, *p.script_args) output = None if processed is not None: output = processed.images diff --git a/modules/lora/networks.py b/modules/lora/networks.py index a38945072..df7778ead 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -314,22 +314,29 @@ def network_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.n if weights_backup is None and wanted_names != (): # pylint: disable=C1803 weight = getattr(self, 'weight', None) self.network_weights_backup = None - if shared.opts.lora_fuse_diffusers: - self.network_weights_backup = True - elif getattr(weight, "quant_type", None) in ['nf4', 'fp4']: + if getattr(weight, "quant_type", None) in ['nf4', 'fp4']: if bnb is None: bnb = model_quant.load_bnb('Load network: type=LoRA', silent=True) if bnb is not None: with devices.inference_context(): - self.network_weights_backup = bnb.functional.dequantize_4bit(weight, quant_state=weight.quant_state, quant_type=weight.quant_type, blocksize=weight.blocksize,) + if shared.opts.lora_fuse_diffusers: + self.network_weights_backup = True + else: + self.network_weights_backup = bnb.functional.dequantize_4bit(weight, quant_state=weight.quant_state, quant_type=weight.quant_type, blocksize=weight.blocksize,) self.quant_state = weight.quant_state self.quant_type = weight.quant_type self.blocksize = weight.blocksize else: - weights_backup = weight.clone() - self.network_weights_backup = weights_backup.to(devices.cpu) + if shared.opts.lora_fuse_diffusers: + self.network_weights_backup = True + else: + weights_backup = weight.clone() + self.network_weights_backup = weights_backup.to(devices.cpu) else: - self.network_weights_backup = weight.clone().to(devices.cpu) + if shared.opts.lora_fuse_diffusers: + self.network_weights_backup = True + else: + self.network_weights_backup = weight.clone().to(devices.cpu) bias_backup = getattr(self, "network_bias_backup", None) if bias_backup is None: @@ -408,13 +415,20 @@ def network_apply_direct(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn. if updown is not None: if deactivate: updown *= -1 - try: - new_weight = self.weight.to(devices.device) + updown.to(devices.device) - except Exception: - new_weight = self.weight + updown if getattr(self, "quant_type", None) in ['nf4', 'fp4'] and bnb is not None: - self.weight = bnb.nn.Params4bit(new_weight, quant_state=self.quant_state, quant_type=self.quant_type, blocksize=self.blocksize) + try: # TODO lora-direct with bnb + weight = bnb.functional.dequantize_4bit(self.weight, quant_state=self.quant_state, quant_type=self.quant_type, blocksize=self.blocksize) + new_weight = weight.to(devices.device) + updown.to(devices.device) + self.weight = bnb.nn.Params4bit(new_weight, quant_state=self.quant_state, quant_type=self.quant_type, blocksize=self.blocksize) + except Exception: + # shared.log.error(f'Load network: type=LoRA quant=bnb type={self.quant_type} state={self.quant_state} blocksize={self.blocksize} {e}') + extra_network_lora.errors['bnb'] = extra_network_lora.errors.get('bnb', 0) + 1 + new_weight = None else: + try: + new_weight = self.weight.to(devices.device) + updown.to(devices.device) + except Exception: + new_weight = self.weight + updown self.weight = torch.nn.Parameter(new_weight, requires_grad=False) del new_weight if hasattr(self, "qweight") and hasattr(self, "freeze"): diff --git a/modules/scripts.py b/modules/scripts.py index cf2cf25b9..029d0db79 100644 --- a/modules/scripts.py +++ b/modules/scripts.py @@ -288,6 +288,7 @@ def register_scripts_from_module(module, scriptfile): current_basedir = paths.script_path t.record(os.path.basename(scriptfile.basedir) if scriptfile.basedir != paths.script_path else scriptfile.filename) sys.path = syspath + global scripts_txt2img, scripts_img2img, scripts_control, scripts_postproc # pylint: disable=global-statement scripts_txt2img = ScriptRunner() scripts_img2img = ScriptRunner() From c60c87bff8e3ffe04c5895402474c79d7dbc04cc Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 20 Dec 2024 18:48:06 -0500 Subject: [PATCH 121/249] linting Signed-off-by: Vladimir Mandic --- TODO.md | 17 +++++++++++++++-- extensions-builtin/Lora/lora_extract.py | 13 ------------- installer.py | 2 +- modules/lora/lora_extract.py | 2 +- modules/model_flux.py | 4 ++-- modules/model_sana.py | 2 +- modules/model_sd3.py | 2 +- modules/processing_class.py | 2 +- modules/sd_models.py | 2 +- modules/sd_models_compile.py | 2 +- modules/textual_inversion/textual_inversion.py | 2 +- 11 files changed, 25 insertions(+), 25 deletions(-) diff --git a/TODO.md b/TODO.md index fafbea8c3..ad43e6d06 100644 --- a/TODO.md +++ b/TODO.md @@ -10,10 +10,8 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma ## Future Candidates -- SD35 IPAdapter: - Flux IPAdapter: - Flux NF4: -- LTX-Video: - GGUF: ## Other @@ -21,3 +19,18 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma - IPAdapter negative: - Control API enhance scripts compatibility - PixelSmith: + +## Code TODO + +- python 3.12.4 or higher cause a mess with pydantic +- enable ROCm for windows when available +- enable full VAE mode for resize-latent +- remove duplicate mask params +- fix flux loader for civitai nf4 models +- implement model in-memory caching +- hypertile vae breaks for diffusers when using non-standard sizes +- forcing reloading entire model as loading transformers only leads to massive memory usage +- lora-direct with bnb +- make lora for quantized flux +- control script process +- monkey-patch for modernui missing tabs.select event diff --git a/extensions-builtin/Lora/lora_extract.py b/extensions-builtin/Lora/lora_extract.py index c2e0a275b..1d92f3c6e 100644 --- a/extensions-builtin/Lora/lora_extract.py +++ b/extensions-builtin/Lora/lora_extract.py @@ -182,19 +182,6 @@ def make_lora(fn, maxrank, auto_rank, rank_ratio, modules, overwrite): progress.remove_task(task) t3 = time.time() - # TODO: Handle quant for Flux - # if 'te' in modules and getattr(shared.sd_model, 'transformer', None) is not None: - # for name, module in shared.sd_model.transformer.named_modules(): - # if "norm" in name and "linear" not in name: - # continue - # weights_backup = getattr(module, "network_weights_backup", None) - # if weights_backup is None: - # continue - # module.svdhandler = SVDHandler() - # module.svdhandler.network_name = "lora_transformer_" + name.replace(".", "_") - # module.svdhandler.decompose(module.weight, weights_backup) - # module.svdhandler.findrank(rank, rank_ratio) - lora_state_dict = {} for sub in ['text_encoder', 'text_encoder_2', 'unet', 'transformer']: submodel = getattr(shared.sd_model, sub, None) diff --git a/installer.py b/installer.py index 40ba7cea0..d186cc5ac 100644 --- a/installer.py +++ b/installer.py @@ -552,7 +552,7 @@ def install_rocm_zluda(): log.info(msg) torch_command = '' if sys.platform == "win32": - # TODO after ROCm for Windows is released + # TODO enable ROCm for windows when available if args.device_id is not None: if os.environ.get('HIP_VISIBLE_DEVICES', None) is not None: diff --git a/modules/lora/lora_extract.py b/modules/lora/lora_extract.py index c2e0a275b..c7deb2530 100644 --- a/modules/lora/lora_extract.py +++ b/modules/lora/lora_extract.py @@ -182,7 +182,7 @@ def make_lora(fn, maxrank, auto_rank, rank_ratio, modules, overwrite): progress.remove_task(task) t3 = time.time() - # TODO: Handle quant for Flux + # TODO: make lora for quantized flux # if 'te' in modules and getattr(shared.sd_model, 'transformer', None) is not None: # for name, module in shared.sd_model.transformer.named_modules(): # if "norm" in name and "linear" not in name: diff --git a/modules/model_flux.py b/modules/model_flux.py index ac1370ef7..759fc9b1e 100644 --- a/modules/model_flux.py +++ b/modules/model_flux.py @@ -149,7 +149,7 @@ def load_quants(kwargs, repo_id, cache_dir): quant_args = model_quant.create_bnb_config(quant_args) quant_args = model_quant.create_ao_config(quant_args) if not quant_args: - return + return kwargs model_quant.load_bnb(f'Load model: type=FLUX quant={quant_args}') if 'Model' in shared.opts.bnb_quantization and 'transformer' not in kwargs: kwargs['transformer'] = diffusers.FluxTransformer2DModel.from_pretrained(repo_id, subfolder="transformer", cache_dir=cache_dir, torch_dtype=devices.dtype, **quant_args) @@ -208,7 +208,7 @@ def load_transformer(file_path): # triggered by opts.sd_unet change _transformer, _text_encoder_2 = load_flux_bnb(file_path, diffusers_load_config) if _transformer is not None: transformer = _transformer - elif 'nf4' in quant: # TODO right now this is not working for civitai published nf4 models + elif 'nf4' in quant: # TODO fix flux loader for civitai nf4 models from modules.model_flux_nf4 import load_flux_nf4 _transformer, _text_encoder_2 = load_flux_nf4(file_path) if _transformer is not None: diff --git a/modules/model_sana.py b/modules/model_sana.py index 414f9b74d..b9f56c7c6 100644 --- a/modules/model_sana.py +++ b/modules/model_sana.py @@ -13,7 +13,7 @@ def load_quants(kwargs, repo_id, cache_dir): quant_args = model_quant.create_ao_config(quant_args) load_args = kwargs.copy() if not quant_args: - return + return kwargs model_quant.load_bnb(f'Load model: type=SD3 quant={quant_args} args={load_args}') if 'Model' in shared.opts.bnb_quantization and 'transformer' not in kwargs: kwargs['transformer'] = diffusers.models.SanaTransformer2DModel.from_pretrained(repo_id, subfolder="transformer", cache_dir=cache_dir, **load_args, **quant_args) diff --git a/modules/model_sd3.py b/modules/model_sd3.py index 2842661bd..e72ecd75c 100644 --- a/modules/model_sd3.py +++ b/modules/model_sd3.py @@ -55,7 +55,7 @@ def load_quants(kwargs, repo_id, cache_dir): quant_args = model_quant.create_bnb_config(quant_args) quant_args = model_quant.create_ao_config(quant_args) if not quant_args: - return + return kwargs model_quant.load_bnb(f'Load model: type=SD3 quant={quant_args}') if 'Model' in shared.opts.bnb_quantization and 'transformer' not in kwargs: kwargs['transformer'] = diffusers.SD3Transformer2DModel.from_pretrained(repo_id, subfolder="transformer", cache_dir=cache_dir, torch_dtype=devices.dtype, **quant_args) diff --git a/modules/processing_class.py b/modules/processing_class.py index 7a7d9cd36..45ca70815 100644 --- a/modules/processing_class.py +++ b/modules/processing_class.py @@ -170,7 +170,7 @@ def __init__(self, self.image_cfg_scale = image_cfg_scale self.scale_by = scale_by self.mask = mask - self.image_mask = mask # TODO duplciate mask params + self.image_mask = mask # TODO remove duplicate mask params self.latent_mask = latent_mask self.mask_blur = mask_blur self.inpainting_fill = inpainting_fill diff --git a/modules/sd_models.py b/modules/sd_models.py index 5939bccbd..f7edc6ddf 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -1495,7 +1495,7 @@ def reload_model_weights(sd_model=None, info=None, reuse_dict=False, op='model', unload_model_weights(op=op) sd_model = None timer = Timer() - # TODO implement caching after diffusers implement state_dict loading + # TODO implement model in-memory caching state_dict = get_checkpoint_state_dict(checkpoint_info, timer) if not shared.native else None checkpoint_config = sd_models_config.find_checkpoint_config(state_dict, checkpoint_info) timer.record("config") diff --git a/modules/sd_models_compile.py b/modules/sd_models_compile.py index 16540019c..20a7d7de2 100644 --- a/modules/sd_models_compile.py +++ b/modules/sd_models_compile.py @@ -515,7 +515,7 @@ def torchao_quantization(sd_model): if fn is None: shared.log.error(f"Quantization: type=TorchAO type={shared.opts.torchao_quantization_type} not supported") return sd_model - def torchao_model(model, op=None, sd_model=None): + def torchao_model(model, op=None, sd_model=None): # pylint: disable=unused-argument q.quantize_(model, fn(), device=devices.device) return model diff --git a/modules/textual_inversion/textual_inversion.py b/modules/textual_inversion/textual_inversion.py index 1c6923285..de12021a5 100644 --- a/modules/textual_inversion/textual_inversion.py +++ b/modules/textual_inversion/textual_inversion.py @@ -4,7 +4,7 @@ import torch import safetensors.torch from PIL import Image -from modules import shared, devices, sd_models, errors +from modules import shared, devices, errors from modules.textual_inversion.image_embedding import embedding_from_b64, extract_image_data_embed from modules.files_cache import directory_files, directory_mtime, extension_filter From 431cb059f5cf413036eb2d3b1b385f7a042da711 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 20 Dec 2024 18:58:29 -0500 Subject: [PATCH 122/249] ruff updates Signed-off-by: Vladimir Mandic --- extensions-builtin/Lora/network_lora.py | 4 ++-- modules/devices_mac.py | 4 ++-- modules/images_namegen.py | 4 ++-- modules/lora/network_lora.py | 4 ++-- modules/merging/merge_methods.py | 2 +- modules/processing_args.py | 2 +- modules/processing_helpers.py | 4 ++-- modules/sd_disable_initialization.py | 2 +- modules/sd_hijack_unet.py | 2 +- scripts/ltxvideo.py | 2 +- scripts/regional_prompting.py | 2 +- 11 files changed, 16 insertions(+), 16 deletions(-) diff --git a/extensions-builtin/Lora/network_lora.py b/extensions-builtin/Lora/network_lora.py index 8ebda2e22..a410a8e3b 100644 --- a/extensions-builtin/Lora/network_lora.py +++ b/extensions-builtin/Lora/network_lora.py @@ -31,7 +31,7 @@ def create_module(self, weights, key, none_ok=False): if is_linear: weight = weight.reshape(weight.shape[0], -1) module = torch.nn.Linear(weight.shape[1], weight.shape[0], bias=False) - elif is_conv and key == "lora_down.weight" or key == "dyn_up": + elif is_conv and (key == "lora_down.weight" or key == "dyn_up"): if len(weight.shape) == 2: weight = weight.reshape(weight.shape[0], -1, 1, 1) if weight.shape[2] != 1 or weight.shape[3] != 1: @@ -40,7 +40,7 @@ def create_module(self, weights, key, none_ok=False): module = torch.nn.Conv2d(weight.shape[1], weight.shape[0], (1, 1), bias=False) elif is_conv and key == "lora_mid.weight": module = torch.nn.Conv2d(weight.shape[1], weight.shape[0], self.sd_module.kernel_size, self.sd_module.stride, self.sd_module.padding, bias=False) - elif is_conv and key == "lora_up.weight" or key == "dyn_down": + elif is_conv and (key == "lora_up.weight" or key == "dyn_down"): module = torch.nn.Conv2d(weight.shape[1], weight.shape[0], (1, 1), bias=False) else: raise AssertionError(f'Lora unsupported: layer={self.network_key} type={type(self.sd_module).__name__}') diff --git a/modules/devices_mac.py b/modules/devices_mac.py index fe7c80f31..2ddc1fc22 100644 --- a/modules/devices_mac.py +++ b/modules/devices_mac.py @@ -24,7 +24,7 @@ def cumsum_fix(input, cumsum_func, *args, **kwargs): # pylint: disable=redefined output_dtype = kwargs.get('dtype', input.dtype) if output_dtype == torch.int64: return cumsum_func(input.cpu(), *args, **kwargs).to(input.device) - elif output_dtype == torch.bool or cumsum_needs_int_fix and (output_dtype == torch.int8 or output_dtype == torch.int16): + elif output_dtype == torch.bool or (cumsum_needs_int_fix and (output_dtype == torch.int8 or output_dtype == torch.int16)): return cumsum_func(input.to(torch.int32), *args, **kwargs).to(torch.int64) return cumsum_func(input, *args, **kwargs) @@ -42,7 +42,7 @@ def cumsum_fix(input, cumsum_func, *args, **kwargs): # pylint: disable=redefined # MPS workaround for https://github.com/pytorch/pytorch/issues/79383 CondFunc('torch.Tensor.to', lambda orig_func, self, *args, **kwargs: orig_func(self.contiguous(), *args, **kwargs), - lambda _, self, *args, **kwargs: self.device.type != 'mps' and (args and isinstance(args[0], torch.device) and args[0].type == 'mps' or isinstance(kwargs.get('device'), torch.device) and kwargs['device'].type == 'mps')) + lambda _, self, *args, **kwargs: self.device.type != 'mps' and ((args and isinstance(args[0], torch.device) and args[0].type == 'mps') or (isinstance(kwargs.get('device'), torch.device) and kwargs['device'].type == 'mps'))) # MPS workaround for https://github.com/pytorch/pytorch/issues/80800 CondFunc('torch.nn.functional.layer_norm', lambda orig_func, *args, **kwargs: orig_func(*([args[0].contiguous()] + list(args[1:])), **kwargs), lambda _, *args, **kwargs: args and isinstance(args[0], torch.Tensor) and args[0].device.type == 'mps') diff --git a/modules/images_namegen.py b/modules/images_namegen.py index bc58f728a..7cad39b92 100644 --- a/modules/images_namegen.py +++ b/modules/images_namegen.py @@ -45,12 +45,12 @@ class FilenameGenerator: 'prompt_hash': lambda self: hashlib.sha256(self.prompt.encode()).hexdigest()[0:8], 'sampler': lambda self: self.p and self.p.sampler_name, - 'seed': lambda self: self.seed and str(self.seed) or '', + 'seed': lambda self: (self.seed and str(self.seed)) or '', 'steps': lambda self: self.p and getattr(self.p, 'steps', 0), 'cfg': lambda self: self.p and getattr(self.p, 'cfg_scale', 0), 'clip_skip': lambda self: self.p and getattr(self.p, 'clip_skip', 0), 'denoising': lambda self: self.p and getattr(self.p, 'denoising_strength', 0), - 'styles': lambda self: self.p and ", ".join([style for style in self.p.styles if not style == "None"]) or "None", + 'styles': lambda self: (self.p and ", ".join([style for style in self.p.styles if not style == "None"])) or "None", 'uuid': lambda self: str(uuid.uuid4()), } default_time_format = '%Y%m%d%H%M%S' diff --git a/modules/lora/network_lora.py b/modules/lora/network_lora.py index 8bf475ebc..3604e059d 100644 --- a/modules/lora/network_lora.py +++ b/modules/lora/network_lora.py @@ -31,7 +31,7 @@ def create_module(self, weights, key, none_ok=False): if is_linear: weight = weight.reshape(weight.shape[0], -1) module = torch.nn.Linear(weight.shape[1], weight.shape[0], bias=False) - elif is_conv and key == "lora_down.weight" or key == "dyn_up": + elif is_conv and (key == "lora_down.weight" or key == "dyn_up"): if len(weight.shape) == 2: weight = weight.reshape(weight.shape[0], -1, 1, 1) if weight.shape[2] != 1 or weight.shape[3] != 1: @@ -40,7 +40,7 @@ def create_module(self, weights, key, none_ok=False): module = torch.nn.Conv2d(weight.shape[1], weight.shape[0], (1, 1), bias=False) elif is_conv and key == "lora_mid.weight": module = torch.nn.Conv2d(weight.shape[1], weight.shape[0], self.sd_module.kernel_size, self.sd_module.stride, self.sd_module.padding, bias=False) - elif is_conv and key == "lora_up.weight" or key == "dyn_down": + elif is_conv and (key == "lora_up.weight" or key == "dyn_down"): module = torch.nn.Conv2d(weight.shape[1], weight.shape[0], (1, 1), bias=False) else: raise AssertionError(f'Lora unsupported: layer={self.network_key} type={type(self.sd_module).__name__}') diff --git a/modules/merging/merge_methods.py b/modules/merging/merge_methods.py index 3f704c20f..ce196b60c 100644 --- a/modules/merging/merge_methods.py +++ b/modules/merging/merge_methods.py @@ -4,7 +4,7 @@ import torch from torch import Tensor -__all__ = [ +__all__ = [ # noqa: RUF022 "weighted_sum", "weighted_subtraction", "tensor_sum", diff --git a/modules/processing_args.py b/modules/processing_args.py index d0afb6722..3709368ee 100644 --- a/modules/processing_args.py +++ b/modules/processing_args.py @@ -23,7 +23,7 @@ def task_specific_kwargs(p, model): if isinstance(p.init_images[0], str): p.init_images = [helpers.decode_base64_to_image(i, quiet=True) for i in p.init_images] p.init_images = [i.convert('RGB') if i.mode != 'RGB' else i for i in p.init_images] - if sd_models.get_diffusers_task(model) == sd_models.DiffusersTaskType.TEXT_2_IMAGE or len(getattr(p, 'init_images', [])) == 0 and not is_img2img_model: + if (sd_models.get_diffusers_task(model) == sd_models.DiffusersTaskType.TEXT_2_IMAGE or len(getattr(p, 'init_images', [])) == 0) and not is_img2img_model: p.ops.append('txt2img') if hasattr(p, 'width') and hasattr(p, 'height'): task_args = { diff --git a/modules/processing_helpers.py b/modules/processing_helpers.py index 5d2661cc2..304e2c211 100644 --- a/modules/processing_helpers.py +++ b/modules/processing_helpers.py @@ -159,7 +159,7 @@ def create_random_tensors(shape, seeds, subseeds=None, subseed_strength=0.0, see # enables the generation of additional tensors with noise that the sampler will use during its processing. # Using those pre-generated tensors instead of simple torch.randn allows a batch with seeds [100, 101] to # produce the same images as with two batches [100], [101]. - if p is not None and p.sampler is not None and (len(seeds) > 1 and shared.opts.enable_batch_seeds or eta_noise_seed_delta > 0): + if p is not None and p.sampler is not None and ((len(seeds) > 1 and shared.opts.enable_batch_seeds) or (eta_noise_seed_delta > 0)): sampler_noises = [[] for _ in range(p.sampler.number_of_needed_noises(p))] else: sampler_noises = None @@ -414,7 +414,7 @@ def resize_hires(p, latents): # input=latents output=pil if not latent_upscaler if latent_upscaler is not None: return torch.nn.functional.interpolate(latents, size=(p.hr_upscale_to_y // 8, p.hr_upscale_to_x // 8), mode=latent_upscaler["mode"], antialias=latent_upscaler["antialias"]) first_pass_images = processing_vae.vae_decode(latents=latents, model=shared.sd_model, full_quality=p.full_quality, output_type='pil', width=p.width, height=p.height) - if p.hr_upscale_to_x == 0 or p.hr_upscale_to_y == 0 and hasattr(p, 'init_hr'): + if p.hr_upscale_to_x == 0 or (p.hr_upscale_to_y == 0 and hasattr(p, 'init_hr')): shared.log.error('Hires: missing upscaling dimensions') return first_pass_images resized_images = [] diff --git a/modules/sd_disable_initialization.py b/modules/sd_disable_initialization.py index e9ac1be92..688af0f34 100644 --- a/modules/sd_disable_initialization.py +++ b/modules/sd_disable_initialization.py @@ -53,7 +53,7 @@ def transformers_modeling_utils_load_pretrained_model(*args, **kwargs): def transformers_utils_hub_get_file_from_cache(original, url, *args, **kwargs): # this file is always 404, prevent making request - if url == 'https://huggingface.co/openai/clip-vit-large-patch14/resolve/main/added_tokens.json' or url == 'openai/clip-vit-large-patch14' and args[0] == 'added_tokens.json': + if (url == 'https://huggingface.co/openai/clip-vit-large-patch14/resolve/main/added_tokens.json' or url == 'openai/clip-vit-large-patch14') and args[0] == 'added_tokens.json': return None try: diff --git a/modules/sd_hijack_unet.py b/modules/sd_hijack_unet.py index d8d356071..66040c5aa 100644 --- a/modules/sd_hijack_unet.py +++ b/modules/sd_hijack_unet.py @@ -70,7 +70,7 @@ def hijack_ddpm_edit(): if version.parse(torch.__version__) <= version.parse("1.13.2") or torch.cuda.is_available(): CondFunc('ldm.modules.diffusionmodules.util.GroupNorm32.forward', lambda orig_func, self, *args, **kwargs: orig_func(self.float(), *args, **kwargs), unet_needs_upcast) CondFunc('ldm.modules.attention.GEGLU.forward', lambda orig_func, self, x: orig_func(self.float(), x.float()).to(devices.dtype_unet), unet_needs_upcast) - CondFunc('open_clip.transformer.ResidualAttentionBlock.__init__', lambda orig_func, *args, **kwargs: kwargs.update({'act_layer': GELUHijack}) and False or orig_func(*args, **kwargs), lambda _, *args, **kwargs: kwargs.get('act_layer') is None or kwargs['act_layer'] == torch.nn.GELU) + CondFunc('open_clip.transformer.ResidualAttentionBlock.__init__', lambda orig_func, *args, **kwargs: (kwargs.update({'act_layer': GELUHijack}) and False) or orig_func(*args, **kwargs), lambda _, *args, **kwargs: kwargs.get('act_layer') is None or kwargs['act_layer'] == torch.nn.GELU) first_stage_cond = lambda _, self, *args, **kwargs: devices.unet_needs_upcast and self.model.diffusion_model.dtype == torch.float16 # pylint: disable=unnecessary-lambda-assignment first_stage_sub = lambda orig_func, self, x, **kwargs: orig_func(self, x.to(devices.dtype_vae), **kwargs) # pylint: disable=unnecessary-lambda-assignment diff --git a/scripts/ltxvideo.py b/scripts/ltxvideo.py index 50530563a..148fd4481 100644 --- a/scripts/ltxvideo.py +++ b/scripts/ltxvideo.py @@ -43,7 +43,7 @@ def run(self, p: processing.StableDiffusionProcessing, num_frames, video_type, d # set params image = getattr(p, 'init_images', None) image = None if image is None or len(image) == 0 else image[0] - if p.width == 0 or p.height == 0 and image is not None: + if (p.width == 0 or p.height == 0) and image is not None: p.width = image.width p.height = image.height num_frames = 8 * int(num_frames // 8) + 1 diff --git a/scripts/regional_prompting.py b/scripts/regional_prompting.py index 48309704e..3d452b0c5 100644 --- a/scripts/regional_prompting.py +++ b/scripts/regional_prompting.py @@ -9,7 +9,7 @@ def hijack_register_modules(self, **kwargs): for name, module in kwargs.items(): register_dict = None - if module is None or isinstance(module, (tuple, list)) and module[0] is None: + if module is None or (isinstance(module, (tuple, list)) and module[0] is None): register_dict = {name: (None, None)} elif isinstance(module, bool): pass From 0d124d5d5fd3de7d0ae88941e54b2ff2efddb1f4 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 20 Dec 2024 19:23:06 -0500 Subject: [PATCH 123/249] update changelog Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ebf17d6b2..a07fda734 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ And a lot of **Control** and **IPAdapter** goodies Plus couple of new integrated workflows such as [FreeScale](https://github.com/ali-vilab/FreeScale) and [Style Aligned Image Generation](https://style-aligned-gen.github.io/) And it wouldn't be a *Xmass edition* without couple of custom themes: *Snowflake* and *Elf-Green*! +All-in-all, we're around ~160 commits worth of updates, check changelog for full list [ReadMe](https://github.com/vladmandic/automatic/blob/master/README.md) | [ChangeLog](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) | [Docs](https://vladmandic.github.io/sdnext-docs/) | [WiKi](https://github.com/vladmandic/automatic/wiki) | [Discord](https://discord.com/invite/sd-next-federal-batch-inspectors-1101998836328697867) @@ -555,7 +556,7 @@ And there are also other goodies like multiple *XYZ grid* improvements, addition - xyz grid support for sampler options - metadata updates for sampler options - modernui updates for sampler options - - *note* sampler options defaults are not save in ui settings, they are saved in server settings + - *note* sampler options defaults are not saved in ui settings, they are saved in server settings to apply your defaults, set ui values and apply via *system -> settings -> apply settings* *sampler options*: From 49b53a5fdf7af8164ab510dd4e4d7dff79105ad9 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 21 Dec 2024 09:19:51 -0500 Subject: [PATCH 124/249] switch gguf loader Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 5 ++++- modules/ggml/__init__.py | 45 +++++++++++++++++++++++++++++++++++----- modules/model_flux.py | 18 ++++++++++++---- modules/model_sd3.py | 13 +++++++++--- modules/model_te.py | 19 +++-------------- modules/sd_detect.py | 11 ++++++++++ 6 files changed, 82 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a07fda734..d31807583 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ ### SD.Next Xmass edition: *What's new?* While we have several new supported models, workflows and tools, this release is primarily about *quality-of-life improvements*: -- New memory management engine: list of changes that went into this one is long: changes to GPU offloading, LoRA loader, system memory management, on-the-fly quantization, etc. +- New memory management engine: list of changes that went into this one is long: changes to GPU offloading, brand new LoRA loader, system memory management, on-the-fly quantization, improved gguf loader, etc. but main goal is enabling modern large models to run on standard consumer GPUs without performance hits typically associated with aggressive memory swapping and needs for constant manual tweaks - New [documentation website](https://vladmandic.github.io/sdnext-docs/) @@ -165,6 +165,7 @@ All-in-all, we're around ~160 commits worth of updates, check changelog for full - `BitsAndBytes` with 3 float-based quantization schemes - `Optimium.Quanto` with 3 int-based and 2 float-based quantizations schemes - `GGUF` with pre-quantized weights + - Switch `GGUF` loader from custom to diffuser native - **IPEX**: update to IPEX 2.5.10+xpu - **OpenVINO**: update to 2024.5.0 - **Sampler** improvements @@ -202,6 +203,8 @@ All-in-all, we're around ~160 commits worth of updates, check changelog for full - uninstall conflicting `wandb` package - dont skip diffusers version check if quick is specified - notify on torch install +- detect pipeline fro diffusers folder-style model +- do not recast flux quants ## Update for 2024-11-21 diff --git a/modules/ggml/__init__.py b/modules/ggml/__init__.py index acac6057c..44721d846 100644 --- a/modules/ggml/__init__.py +++ b/modules/ggml/__init__.py @@ -1,11 +1,35 @@ -from pathlib import Path +import os +import time import torch -import gguf -from .gguf_utils import TORCH_COMPATIBLE_QTYPES -from .gguf_tensor import GGMLTensor +import diffusers +import transformers -def load_gguf_state_dict(path: str, compute_dtype: torch.dtype) -> dict[str, GGMLTensor]: +def install_gguf(): + # pip install git+https://github.com/junejae/transformers@feature/t5-gguf + # https://github.com/ggerganov/llama.cpp/issues/9566 + from installer import install + install('gguf', quiet=True) + import importlib + import gguf + from modules import shared + scripts_dir = os.path.join(os.path.dirname(gguf.__file__), '..', 'scripts') + if os.path.exists(scripts_dir): + os.rename(scripts_dir, scripts_dir + str(time.time())) + # monkey patch transformers/diffusers so they detect newly installed gguf pacakge correctly + ver = importlib.metadata.version('gguf') + transformers.utils.import_utils._is_gguf_available = True # pylint: disable=protected-access + transformers.utils.import_utils._gguf_version = ver # pylint: disable=protected-access + diffusers.utils.import_utils._is_gguf_available = True # pylint: disable=protected-access + diffusers.utils.import_utils._gguf_version = ver # pylint: disable=protected-access + shared.log.debug(f'Load GGUF: version={ver}') + return gguf + + +def load_gguf_state_dict(path: str, compute_dtype: torch.dtype) -> dict: + gguf = install_gguf() + from .gguf_utils import TORCH_COMPATIBLE_QTYPES + from .gguf_tensor import GGMLTensor sd: dict[str, GGMLTensor] = {} stats = {} reader = gguf.GGUFReader(path) @@ -19,3 +43,14 @@ def load_gguf_state_dict(path: str, compute_dtype: torch.dtype) -> dict[str, GGM stats[tensor.tensor_type.name] = 0 stats[tensor.tensor_type.name] += 1 return sd, stats + + +def load_gguf(path, cls, compute_dtype: torch.dtype): + _gguf = install_gguf() + module = cls.from_single_file( + path, + quantization_config = diffusers.GGUFQuantizationConfig(compute_dtype=compute_dtype), + torch_dtype=compute_dtype, + ) + module.gguf = 'gguf' + return module diff --git a/modules/model_flux.py b/modules/model_flux.py index 759fc9b1e..0a13a4d46 100644 --- a/modules/model_flux.py +++ b/modules/model_flux.py @@ -160,9 +160,10 @@ def load_quants(kwargs, repo_id, cache_dir): return kwargs +""" def load_flux_gguf(file_path): transformer = None - model_te.install_gguf() + ggml.install_gguf() from accelerate import init_empty_weights from diffusers.loaders.single_file_utils import convert_flux_transformer_checkpoint_to_diffusers from modules import ggml, sd_hijack_accelerate @@ -180,9 +181,11 @@ def load_flux_gguf(file_path): continue applied += 1 sd_hijack_accelerate.hijack_set_module_tensor_simple(transformer, tensor_name=param_name, value=param, device=0) + transformer.gguf = 'gguf' state_dict[param_name] = None shared.log.debug(f'Load model: type=Unet/Transformer applied={applied} skipped={skipped} stats={stats}') return transformer, None +""" def load_transformer(file_path): # triggered by opts.sd_unet change @@ -197,7 +200,9 @@ def load_transformer(file_path): # triggered by opts.sd_unet change } shared.log.info(f'Load module: type=UNet/Transformer file="{file_path}" offload={shared.opts.diffusers_offload_mode} quant={quant} dtype={devices.dtype}') if 'gguf' in file_path.lower(): - _transformer, _text_encoder_2 = load_flux_gguf(file_path) + # _transformer, _text_encoder_2 = load_flux_gguf(file_path) + from modules import ggml + _transformer = ggml.load_gguf(file_path, cls=diffusers.FluxTransformer2DModel, compute_dtype=devices.dtype) if _transformer is not None: transformer = _transformer elif quant == 'qint8' or quant == 'qint4': @@ -336,9 +341,14 @@ def load_flux(checkpoint_info, diffusers_load_config): # triggered by opts.sd_ch cls = diffusers.FluxPipeline shared.log.debug(f'Load model: type=FLUX cls={cls.__name__} preloaded={list(kwargs)} revision={diffusers_load_config.get("revision", None)}') for c in kwargs: + if getattr(kwargs[c], 'quantization_method', None) is not None or getattr(kwargs[c], 'gguf', None) is not None: + shared.log.debug(f'Load model: type=FLUX component={c} dtype={kwargs[c].dtype} quant={getattr(kwargs[c], 'quantization_method', None) or getattr(kwargs[c], 'gguf', None)}') if kwargs[c].dtype == torch.float32 and devices.dtype != torch.float32: - shared.log.warning(f'Load model: type=FLUX component={c} dtype={kwargs[c].dtype} cast dtype={devices.dtype} recast') - kwargs[c] = kwargs[c].to(dtype=devices.dtype) + try: + kwargs[c] = kwargs[c].to(dtype=devices.dtype) + shared.log.warning(f'Load model: type=FLUX component={c} dtype={kwargs[c].dtype} cast dtype={devices.dtype} recast') + except Exception: + pass allow_quant = 'gguf' not in (sd_unet.loaded_unet or '') fn = checkpoint_info.path diff --git a/modules/model_sd3.py b/modules/model_sd3.py index e72ecd75c..36dfbd9b4 100644 --- a/modules/model_sd3.py +++ b/modules/model_sd3.py @@ -13,7 +13,9 @@ def load_overrides(kwargs, cache_dir): sd_unet.loaded_unet = shared.opts.sd_unet shared.log.debug(f'Load model: type=SD3 unet="{shared.opts.sd_unet}" fmt=safetensors') elif fn.endswith('.gguf'): - kwargs = load_gguf(kwargs, fn) + from modules import ggml + # kwargs = load_gguf(kwargs, fn) + kwargs['transformer'] = ggml.load_gguf(fn, cls=diffusers.SD3Transformer2DModel, compute_dtype=devices.dtype) sd_unet.loaded_unet = shared.opts.sd_unet shared.log.debug(f'Load model: type=SD3 unet="{shared.opts.sd_unet}" fmt=gguf') except Exception as e: @@ -90,8 +92,9 @@ def load_missing(kwargs, fn, cache_dir): return kwargs +""" def load_gguf(kwargs, fn): - model_te.install_gguf() + ggml.install_gguf() from accelerate import init_empty_weights from diffusers.loaders.single_file_utils import convert_sd3_transformer_checkpoint_to_diffusers from modules import ggml, sd_hijack_accelerate @@ -108,10 +111,12 @@ def load_gguf(kwargs, fn): continue applied += 1 sd_hijack_accelerate.hijack_set_module_tensor_simple(transformer, tensor_name=param_name, value=param, device=0) + transformer.gguf = 'gguf' state_dict[param_name] = None shared.log.debug(f'Load model: type=Unet/Transformer applied={applied} skipped={skipped} stats={stats} compute={devices.dtype}') kwargs['transformer'] = transformer return kwargs +""" def load_sd3(checkpoint_info, cache_dir=None, config=None): @@ -139,7 +144,9 @@ def load_sd3(checkpoint_info, cache_dir=None, config=None): # kwargs = load_missing(kwargs, fn, cache_dir) repo_id = fn elif fn.endswith('.gguf'): - kwargs = load_gguf(kwargs, fn) + from modules import ggml + kwargs['transformer'] = ggml.load_gguf(fn, cls=diffusers.SD3Transformer2DModel, compute_dtype=devices.dtype) + # kwargs = load_gguf(kwargs, fn) kwargs = load_missing(kwargs, fn, cache_dir) kwargs['variant'] = 'fp16' else: diff --git a/modules/model_te.py b/modules/model_te.py index 8227ba3e8..16bb6d222 100644 --- a/modules/model_te.py +++ b/modules/model_te.py @@ -12,20 +12,6 @@ loaded_te = None -def install_gguf(): - # pip install git+https://github.com/junejae/transformers@feature/t5-gguf - install('gguf', quiet=True) - # https://github.com/ggerganov/llama.cpp/issues/9566 - import gguf - scripts_dir = os.path.join(os.path.dirname(gguf.__file__), '..', 'scripts') - if os.path.exists(scripts_dir): - os.rename(scripts_dir, scripts_dir + '_gguf') - # monkey patch transformers so they detect gguf pacakge correctly - import importlib - transformers.utils.import_utils._is_gguf_available = True # pylint: disable=protected-access - transformers.utils.import_utils._gguf_version = importlib.metadata.version('gguf') # pylint: disable=protected-access - - def load_t5(name=None, cache_dir=None): global loaded_te # pylint: disable=global-statement if name is None: @@ -34,8 +20,9 @@ def load_t5(name=None, cache_dir=None): modelloader.hf_login() repo_id = 'stabilityai/stable-diffusion-3-medium-diffusers' fn = te_dict.get(name) if name in te_dict else None - if fn is not None and 'gguf' in name.lower(): - install_gguf() + if fn is not None and name.lower().endswith('gguf'): + from modules import ggml + ggml.install_gguf() with open(os.path.join('configs', 'flux', 'text_encoder_2', 'config.json'), encoding='utf8') as f: t5_config = transformers.T5Config(**json.load(f)) t5 = transformers.T5EncoderModel.from_pretrained(None, gguf_file=fn, config=t5_config, device_map="auto", cache_dir=cache_dir, torch_dtype=devices.dtype) diff --git a/modules/sd_detect.py b/modules/sd_detect.py index 1931b9077..fe3d325c9 100644 --- a/modules/sd_detect.py +++ b/modules/sd_detect.py @@ -96,6 +96,17 @@ def detect_pipeline(f: str, op: str = 'model', warning=True, quiet=False): guess = 'FLUX' if size > 11000 and size < 16000: warn(f'Model detected as FLUX UNET model, but attempting to load a base model: {op}={f} size={size} MB') + # guess for diffusers + index = os.path.join(f, 'model_index.json') + if os.path.exists(index) and os.path.isfile(index): + index = shared.readfile(index, silent=True) + cls = index.get('_class_name', None) + if cls is not None: + pipeline = getattr(diffusers, cls) + if 'Flux' in pipeline.__name__: + guess = 'FLUX' + if 'StableDiffusion3' in pipeline.__name__: + guess = 'Stable Diffusion 3' # switch for specific variant if guess == 'Stable Diffusion' and 'inpaint' in f.lower(): guess = 'Stable Diffusion Inpaint' From 226e5131681b71c80b29980f7836c777b2c85f4a Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 21 Dec 2024 10:16:33 -0500 Subject: [PATCH 125/249] batch image processing use captions Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 8 ++-- modules/images.py | 2 + modules/img2img.py | 93 +++++++++++++++++++++---------------- modules/processing_class.py | 8 ++-- modules/ui_img2img.py | 9 ++-- 5 files changed, 70 insertions(+), 50 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d31807583..f62738950 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,8 @@ # Change Log for SD.Next -## Update for 2024-12-20 +## Update for 2024-12-21 -### Highlights for 2024-12-20 +### Highlights for 2024-12-21 ### SD.Next Xmass edition: *What's new?* @@ -29,7 +29,7 @@ All-in-all, we're around ~160 commits worth of updates, check changelog for full [ReadMe](https://github.com/vladmandic/automatic/blob/master/README.md) | [ChangeLog](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) | [Docs](https://vladmandic.github.io/sdnext-docs/) | [WiKi](https://github.com/vladmandic/automatic/wiki) | [Discord](https://discord.com/invite/sd-next-federal-batch-inspectors-1101998836328697867) -## Details for 2024-12-20 +## Details for 2024-12-21 ### New models and integrations @@ -155,6 +155,8 @@ All-in-all, we're around ~160 commits worth of updates, check changelog for full - settings: reorganized and simplified - browser -> server logging framework - add addtional themes: `black-reimagined`, thanks @Artheriax +- **Batch** + - image batch processing will use caption files if they exist instead of default prompt ### Updates diff --git a/modules/images.py b/modules/images.py index 4a32d6d9c..657a51193 100644 --- a/modules/images.py +++ b/modules/images.py @@ -275,6 +275,8 @@ def safe_decode_string(s: bytes): def read_info_from_image(image: Image, watermark: bool = False): + if image is None: + return '', {} items = image.info or {} geninfo = items.pop('parameters', None) or items.pop('UserComment', None) if geninfo is not None and len(geninfo) > 0: diff --git a/modules/img2img.py b/modules/img2img.py index 077df1259..2e3eca54d 100644 --- a/modules/img2img.py +++ b/modules/img2img.py @@ -1,6 +1,7 @@ import os import itertools # SBM Batch frames import numpy as np +import filetype from PIL import Image, ImageOps, ImageFilter, ImageEnhance, ImageChops, UnidentifiedImageError import modules.scripts from modules import shared, processing, images @@ -8,7 +9,6 @@ from modules.ui import plaintext_to_html from modules.memstats import memory_stats - debug = shared.log.trace if os.environ.get('SD_PROCESS_DEBUG', None) is not None else lambda *args, **kwargs: None debug('Trace: PROCESS') @@ -16,24 +16,25 @@ def process_batch(p, input_files, input_dir, output_dir, inpaint_mask_dir, args): shared.log.debug(f'batch: {input_files}|{input_dir}|{output_dir}|{inpaint_mask_dir}') processing.fix_seed(p) + image_files = [] if input_files is not None and len(input_files) > 0: image_files = [f.name for f in input_files] - else: - if not os.path.isdir(input_dir): - shared.log.error(f"Process batch: directory not found: {input_dir}") - return - image_files = os.listdir(input_dir) - image_files = [os.path.join(input_dir, f) for f in image_files] + image_files = [f for f in image_files if filetype.is_image(f)] + shared.log.info(f'Process batch: input images={len(image_files)}') + elif os.path.isdir(input_dir): + image_files = [os.path.join(input_dir, f) for f in os.listdir(input_dir)] + image_files = [f for f in image_files if filetype.is_image(f)] + shared.log.info(f'Process batch: input folder="{input_dir}" images={len(image_files)}') is_inpaint_batch = False - if inpaint_mask_dir: - inpaint_masks = os.listdir(inpaint_mask_dir) - inpaint_masks = [os.path.join(inpaint_mask_dir, f) for f in inpaint_masks] + if inpaint_mask_dir and os.path.isdir(inpaint_mask_dir): + inpaint_masks = [os.path.join(inpaint_mask_dir, f) for f in os.listdir(inpaint_mask_dir)] + inpaint_masks = [f for f in inpaint_masks if filetype.is_image(f)] is_inpaint_batch = len(inpaint_masks) > 0 - if is_inpaint_batch: - shared.log.info(f"Process batch: inpaint batch masks={len(inpaint_masks)}") + shared.log.info(f'Process batch: mask folder="{input_dir}" images={len(inpaint_masks)}') save_normally = output_dir == '' p.do_not_save_grid = True p.do_not_save_samples = not save_normally + p.default_prompt = p.prompt shared.state.job_count = len(image_files) * p.n_iter if shared.opts.batch_frame_mode: # SBM Frame mode is on, process each image in batch with same seed window_size = p.batch_size @@ -55,14 +56,29 @@ def process_batch(p, input_files, input_dir, output_dir, inpaint_mask_dir, args) for image_file in batch_image_files: try: img = Image.open(image_file) - if p.scale_by != 1: - p.width = int(img.width * p.scale_by) - p.height = int(img.height * p.scale_by) + img = ImageOps.exif_transpose(img) + batch_images.append(img) + # p.init() + p.width = int(img.width * p.scale_by) + p.height = int(img.height * p.scale_by) + caption_file = os.path.splitext(image_file)[0] + '.txt' + prompt_type='default' + if os.path.exists(caption_file): + with open(caption_file, 'r', encoding='utf8') as f: + p.prompt = f.read() + prompt_type='file' + else: + p.prompt = p.default_prompt + p.all_prompts = None + p.all_negative_prompts = None + p.all_seeds = None + p.all_subseeds = None + shared.log.debug(f'Process batch: image="{image_file}" prompt={prompt_type} i={i+1}/{len(image_files)}') except UnidentifiedImageError as e: - shared.log.error(f"Image error: {e}") - continue - img = ImageOps.exif_transpose(img) - batch_images.append(img) + shared.log.error(f'Process batch: image="{image_file}" {e}') + if len(batch_images) == 0: + shared.log.warning("Process batch: no images found in batch") + continue batch_images = batch_images * btcrept # Standard mode sends the same image per batchsize. p.init_images = batch_images @@ -81,17 +97,20 @@ def process_batch(p, input_files, input_dir, output_dir, inpaint_mask_dir, args) batch_image_files = batch_image_files * btcrept # List used for naming later. - proc = modules.scripts.scripts_img2img.run(p, *args) - if proc is None: - proc = processing.process_images(p) - for n, (image, image_file) in enumerate(itertools.zip_longest(proc.images,batch_image_files)): + processed = modules.scripts.scripts_img2img.run(p, *args) + if processed is None: + processed = processing.process_images(p) + + for n, (image, image_file) in enumerate(itertools.zip_longest(processed.images, batch_image_files)): + if image is None: + continue basename = '' if shared.opts.use_original_name_batch: forced_filename, ext = os.path.splitext(os.path.basename(image_file)) else: forced_filename = None ext = shared.opts.samples_format - if len(proc.images) > 1: + if len(processed.images) > 1: basename = f'{n + i}' if shared.opts.batch_frame_mode else f'{n}' else: basename = '' @@ -103,7 +122,7 @@ def process_batch(p, input_files, input_dir, output_dir, inpaint_mask_dir, args) for k, v in items.items(): image.info[k] = v images.save_image(image, path=output_dir, basename=basename, seed=None, prompt=None, extension=ext, info=geninfo, short_filename=True, no_prompt=True, grid=False, pnginfo_section_name="extras", existing_info=image.info, forced_filename=forced_filename) - proc = modules.scripts.scripts_img2img.after(p, proc, *args) + processed = modules.scripts.scripts_img2img.after(p, processed, *args) shared.log.debug(f'Processed: images={len(batch_image_files)} memory={memory_stats()} batch') @@ -147,24 +166,20 @@ def img2img(id_task: str, state: str, mode: int, debug(f'img2img: id_task={id_task}|mode={mode}|prompt={prompt}|negative_prompt={negative_prompt}|prompt_styles={prompt_styles}|init_img={init_img}|sketch={sketch}|init_img_with_mask={init_img_with_mask}|inpaint_color_sketch={inpaint_color_sketch}|inpaint_color_sketch_orig={inpaint_color_sketch_orig}|init_img_inpaint={init_img_inpaint}|init_mask_inpaint={init_mask_inpaint}|steps={steps}|sampler_index={sampler_index}||mask_blur={mask_blur}|mask_alpha={mask_alpha}|inpainting_fill={inpainting_fill}|full_quality={full_quality}|detailer={detailer}|tiling={tiling}|hidiffusion={hidiffusion}|n_iter={n_iter}|batch_size={batch_size}|cfg_scale={cfg_scale}|image_cfg_scale={image_cfg_scale}|clip_skip={clip_skip}|denoising_strength={denoising_strength}|seed={seed}|subseed{subseed}|subseed_strength={subseed_strength}|seed_resize_from_h={seed_resize_from_h}|seed_resize_from_w={seed_resize_from_w}|selected_scale_tab={selected_scale_tab}|height={height}|width={width}|scale_by={scale_by}|resize_mode={resize_mode}|resize_name={resize_name}|resize_context={resize_context}|inpaint_full_res={inpaint_full_res}|inpaint_full_res_padding={inpaint_full_res_padding}|inpainting_mask_invert={inpainting_mask_invert}|img2img_batch_files={img2img_batch_files}|img2img_batch_input_dir={img2img_batch_input_dir}|img2img_batch_output_dir={img2img_batch_output_dir}|img2img_batch_inpaint_mask_dir={img2img_batch_inpaint_mask_dir}|override_settings_texts={override_settings_texts}') - if mode == 5: - if img2img_batch_files is None or len(img2img_batch_files) == 0: - shared.log.debug('Init bactch images not set') - elif init_img: - shared.log.debug('Init image not set') - if sampler_index is None: shared.log.warning('Sampler: invalid') sampler_index = 0 + mode = int(mode) + image = None + mask = None override_settings = create_override_settings_dict(override_settings_texts) - if mode == 0: # img2img + if mode == 0: # img2img if init_img is None: return [], '', '', 'Error: init image not provided' image = init_img.convert("RGB") - mask = None - elif mode == 1: # inpaint + elif mode == 1: # inpaint if init_img_with_mask is None: return [], '', '', 'Error: init image with mask not provided' image = init_img_with_mask["image"] @@ -176,8 +191,7 @@ def img2img(id_task: str, state: str, mode: int, if sketch is None: return [], '', '', 'Error: sketch image not provided' image = sketch.convert("RGB") - mask = None - elif mode == 3: # composite + elif mode == 3: # composite if inpaint_color_sketch is None: return [], '', '', 'Error: color sketch image not provided' image = inpaint_color_sketch @@ -188,15 +202,16 @@ def img2img(id_task: str, state: str, mode: int, blur = ImageFilter.GaussianBlur(mask_blur) image = Image.composite(image.filter(blur), orig, mask.filter(blur)) image = image.convert("RGB") - elif mode == 4: # inpaint upload mask + elif mode == 4: # inpaint upload mask if init_img_inpaint is None: return [], '', '', 'Error: inpaint image not provided' image = init_img_inpaint mask = init_mask_inpaint + elif mode == 5: # process batch + pass # handled later else: shared.log.error(f'Image processing unknown mode: {mode}') - image = None - mask = None + if image is not None: image = ImageOps.exif_transpose(image) if selected_scale_tab == 1 and resize_mode != 0: diff --git a/modules/processing_class.py b/modules/processing_class.py index 45ca70815..2f1d589f3 100644 --- a/modules/processing_class.py +++ b/modules/processing_class.py @@ -348,13 +348,15 @@ def __init__(self, **kwargs): debug(f'Process init: mode={self.__class__.__name__} kwargs={kwargs}') # pylint: disable=protected-access super().__init__(**kwargs) - def init(self, all_prompts=None, all_seeds=None, all_subseeds=None): + def init(self, all_prompts=None, all_negative_prompts=None, all_seeds=None, all_subseeds=None): if shared.native: shared.sd_model = sd_models.set_diffuser_pipe(self.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE) - self.width = self.width or 512 - self.height = self.height or 512 + self.width = self.width or 1024 + self.height = self.height or 1024 if all_prompts is not None: self.all_prompts = all_prompts + if all_negative_prompts is not None: + self.all_negative_prompts = all_negative_prompts if all_seeds is not None: self.all_seeds = all_seeds if all_subseeds is not None: diff --git a/modules/ui_img2img.py b/modules/ui_img2img.py index 3c3d63656..45c901c6d 100644 --- a/modules/ui_img2img.py +++ b/modules/ui_img2img.py @@ -102,12 +102,11 @@ def fn_img_composite_change(img, img_composite): init_mask_inpaint = gr.Image(label="Mask", source="upload", interactive=True, type="pil", elem_id="img_inpaint_mask") with gr.TabItem('Batch', id='batch', elem_id="img2img_batch_tab") as tab_batch: - hidden = '
Disabled when launched with --hide-ui-dir-config.' if shared.cmd_opts.hide_ui_dir_config else '' - gr.HTML(f"

Upload images or process images in a directory
Add inpaint batch mask directory to enable inpaint batch processing {hidden}

") + gr.HTML("

Run image processing on upload images or files in a folder
If masks are provided will run inpaint

") img2img_batch_files = gr.Files(label="Batch Process", interactive=True, elem_id="img2img_image_batch") - img2img_batch_input_dir = gr.Textbox(label="Inpaint batch input directory", **shared.hide_dirs, elem_id="img2img_batch_input_dir") - img2img_batch_output_dir = gr.Textbox(label="Inpaint batch output directory", **shared.hide_dirs, elem_id="img2img_batch_output_dir") - img2img_batch_inpaint_mask_dir = gr.Textbox(label="Inpaint batch mask directory", **shared.hide_dirs, elem_id="img2img_batch_inpaint_mask_dir") + img2img_batch_input_dir = gr.Textbox(label="Batch input directory", **shared.hide_dirs, elem_id="img2img_batch_input_dir") + img2img_batch_output_dir = gr.Textbox(label="Batch output directory", **shared.hide_dirs, elem_id="img2img_batch_output_dir") + img2img_batch_inpaint_mask_dir = gr.Textbox(label="Batch mask directory", **shared.hide_dirs, elem_id="img2img_batch_inpaint_mask_dir") img2img_tabs = [tab_img2img, tab_sketch, tab_inpaint, tab_inpaint_color, tab_inpaint_upload, tab_batch] for i, tab in enumerate(img2img_tabs): From fd72a32c6e4be69b44ea4a0469292689df24568f Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 21 Dec 2024 10:36:24 -0500 Subject: [PATCH 126/249] fix Signed-off-by: Vladimir Mandic --- modules/processing_class.py | 4 +--- modules/prompt_parser.py | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/modules/processing_class.py b/modules/processing_class.py index 2f1d589f3..f502e0dcc 100644 --- a/modules/processing_class.py +++ b/modules/processing_class.py @@ -348,15 +348,13 @@ def __init__(self, **kwargs): debug(f'Process init: mode={self.__class__.__name__} kwargs={kwargs}') # pylint: disable=protected-access super().__init__(**kwargs) - def init(self, all_prompts=None, all_negative_prompts=None, all_seeds=None, all_subseeds=None): + def init(self, all_prompts=None, all_seeds=None, all_subseeds=None): if shared.native: shared.sd_model = sd_models.set_diffuser_pipe(self.sd_model, sd_models.DiffusersTaskType.TEXT_2_IMAGE) self.width = self.width or 1024 self.height = self.height or 1024 if all_prompts is not None: self.all_prompts = all_prompts - if all_negative_prompts is not None: - self.all_negative_prompts = all_negative_prompts if all_seeds is not None: self.all_seeds = all_seeds if all_subseeds is not None: diff --git a/modules/prompt_parser.py b/modules/prompt_parser.py index 3a1288097..1eb7a77ee 100644 --- a/modules/prompt_parser.py +++ b/modules/prompt_parser.py @@ -147,7 +147,7 @@ def __default__(self, data, children, meta): def get_schedule(prompt): try: tree = schedule_parser.parse(prompt) - except lark.exceptions.LarkError: + except Exception: return [[steps, prompt]] return [[t, at_step(t, tree)] for t in collect_steps(steps, tree)] From f72d45ae3d619f5910b76648ae398ad1220d0bc7 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 21 Dec 2024 10:56:24 -0500 Subject: [PATCH 127/249] fix xyz with lora none Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 1 + modules/processing_args.py | 5 ++++- modules/sd_samplers.py | 3 ++- scripts/xyz_grid_shared.py | 4 ++-- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f62738950..6c45ff463 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -207,6 +207,7 @@ All-in-all, we're around ~160 commits worth of updates, check changelog for full - notify on torch install - detect pipeline fro diffusers folder-style model - do not recast flux quants +- fix xyz-grid with lora none ## Update for 2024-11-21 diff --git a/modules/processing_args.py b/modules/processing_args.py index 3709368ee..5ca6cfcd3 100644 --- a/modules/processing_args.py +++ b/modules/processing_args.py @@ -320,11 +320,14 @@ def set_pipeline_args(p, model, prompts: list, negative_prompts: list, prompts_2 clean['negative_prompt'] = len(clean['negative_prompt']) clean.pop('generator', None) clean['parser'] = parser - for k, v in clean.items(): + for k, v in clean.copy().items(): if isinstance(v, torch.Tensor) or isinstance(v, np.ndarray): clean[k] = v.shape if isinstance(v, list) and len(v) > 0 and (isinstance(v[0], torch.Tensor) or isinstance(v[0], np.ndarray)): clean[k] = [x.shape for x in v] + if not debug_enabled and k.endswith('_embeds'): + del clean[k] + clean['prompt'] = 'embeds' shared.log.debug(f'Diffuser pipeline: {model.__class__.__name__} task={sd_models.get_diffusers_task(model)} batch={p.iteration + 1}/{p.n_iter}x{p.batch_size} set={clean}') if p.hdr_clamp or p.hdr_maximize or p.hdr_brightness != 0 or p.hdr_color != 0 or p.hdr_sharpen != 0: diff --git a/modules/sd_samplers.py b/modules/sd_samplers.py index 1b1be2d1a..dc58a2419 100644 --- a/modules/sd_samplers.py +++ b/modules/sd_samplers.py @@ -94,7 +94,8 @@ def create_sampler(name, model): if hasattr(model, "prior_pipe") and hasattr(model.prior_pipe, "scheduler"): model.prior_pipe.scheduler = sampler.sampler model.prior_pipe.scheduler.config.clip_sample = False - shared.log.debug(f'Sampler: sampler="{sampler.name}" class="{model.scheduler.__class__.__name__} config={sampler.config}') + clean_config = {k: v for k, v in sampler.config.items() if v is not None and v is not False} + shared.log.debug(f'Sampler: sampler="{sampler.name}" class="{model.scheduler.__class__.__name__} config={clean_config}') return sampler.sampler else: return None diff --git a/scripts/xyz_grid_shared.py b/scripts/xyz_grid_shared.py index 3fc8d32c8..f9bf26c67 100644 --- a/scripts/xyz_grid_shared.py +++ b/scripts/xyz_grid_shared.py @@ -198,12 +198,12 @@ def list_lora(): def apply_lora(p, x, xs): + p.all_prompts = None + p.all_negative_prompts = None if x == 'None': return x = os.path.basename(x) p.prompt = p.prompt + f" " - p.all_prompts = None - p.all_negative_prompts = None shared.log.debug(f'XYZ grid apply LoRA: "{x}"') From 32764cb51d5ae246de2f86f9965d0570ac4d6ef1 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 21 Dec 2024 11:31:08 -0500 Subject: [PATCH 128/249] fix svd i2v Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 1 + modules/sd_models.py | 1 + scripts/stablevideodiffusion.py | 6 +++++- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c45ff463..fdd90e8e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -208,6 +208,7 @@ All-in-all, we're around ~160 commits worth of updates, check changelog for full - detect pipeline fro diffusers folder-style model - do not recast flux quants - fix xyz-grid with lora none +- fix svd image2video ## Update for 2024-11-21 diff --git a/modules/sd_models.py b/modules/sd_models.py index f7edc6ddf..9daccf11f 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -1192,6 +1192,7 @@ def set_diffuser_pipe(pipe, new_pipe_type): 'InstantIRPipeline', 'FluxFillPipeline', 'FluxControlPipeline', + 'StableVideoDiffusionPipeline', ] n = getattr(pipe.__class__, '__name__', '') diff --git a/scripts/stablevideodiffusion.py b/scripts/stablevideodiffusion.py index f8da35b23..127466701 100644 --- a/scripts/stablevideodiffusion.py +++ b/scripts/stablevideodiffusion.py @@ -16,7 +16,7 @@ class Script(scripts.Script): def title(self): - return 'Video: SVD' + return 'Video: Stable Video Diffusion' def show(self, is_img2img): return is_img2img if shared.native else False @@ -75,12 +75,16 @@ def run(self, p: processing.StableDiffusionProcessing, model, num_frames, overri if model_name != model_loaded or c != 'StableVideoDiffusionPipeline': shared.opts.sd_model_checkpoint = model_path sd_models.reload_model_weights() + shared.sd_model = shared.sd_model.to(torch.float32) # TODO svd runs in fp32 # set params if override_resolution: p.width = 1024 p.height = 576 image = images.resize_image(resize_mode=2, im=image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') + else: + p.width = image.width + p.height = image.height p.ops.append('video') p.do_not_save_grid = True p.init_images = [image] From 5cc8f56f0012596b6b56036c914a8ebf7dda49ed Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 21 Dec 2024 14:19:51 -0500 Subject: [PATCH 129/249] fix gallery display during generate Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 1 + javascript/progressBar.js | 8 ++++++-- requirements.txt | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fdd90e8e8..ea57477fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -209,6 +209,7 @@ All-in-all, we're around ~160 commits worth of updates, check changelog for full - do not recast flux quants - fix xyz-grid with lora none - fix svd image2video +- fix gallery display during generate ## Update for 2024-11-21 diff --git a/javascript/progressBar.js b/javascript/progressBar.js index c385fe5db..dfd895f8e 100644 --- a/javascript/progressBar.js +++ b/javascript/progressBar.js @@ -70,9 +70,13 @@ function requestProgress(id_task, progressEl, galleryEl, atEnd = null, onProgres const initLivePreview = () => { if (!parentGallery) return; const footers = Array.from(gradioApp().querySelectorAll('.gallery_footer')); - for (const footer of footers) footer.style.display = 'none'; // remove all footers + for (const footer of footers) { + if (footer.id !== 'gallery_footer') footer.style.display = 'none'; // remove all footers + } const galleries = Array.from(gradioApp().querySelectorAll('.gallery_main')); - for (const gallery of galleries) gallery.style.display = 'none'; // remove all footers + for (const gallery of galleries) { + if (gallery.id !== 'gallery_gallery') gallery.style.display = 'none'; // remove all footers + } livePreview = document.createElement('div'); livePreview.className = 'livePreview'; diff --git a/requirements.txt b/requirements.txt index 100c57dfb..f44f758e2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -52,7 +52,7 @@ numba==0.59.1 protobuf==4.25.3 pytorch_lightning==1.9.4 tokenizers==0.21.0 -transformers==4.47.0 +transformers==4.47.1 urllib3==1.26.19 Pillow==10.4.0 pillow-jxl-plugin==1.3.0 From 8b667b223477d87f3981c06ea6fa0406567b5bfa Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 21 Dec 2024 16:01:35 -0500 Subject: [PATCH 130/249] flux true-scale and flux ipadapters Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 15 +++++++++---- installer.py | 2 +- modules/errors.py | 4 ++-- modules/ipadapter.py | 50 ++++++++++++++++++++++++++++++----------- modules/pag/__init__.py | 11 ++++++--- modules/processing.py | 2 +- scripts/ipadapter.py | 5 ++++- 7 files changed, 64 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea57477fb..90eda3736 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,20 +7,23 @@ ### SD.Next Xmass edition: *What's new?* While we have several new supported models, workflows and tools, this release is primarily about *quality-of-life improvements*: -- New memory management engine: list of changes that went into this one is long: changes to GPU offloading, brand new LoRA loader, system memory management, on-the-fly quantization, improved gguf loader, etc. +- New memory management engine + list of changes that went into this one is long: changes to GPU offloading, brand new LoRA loader, system memory management, on-the-fly quantization, improved gguf loader, etc. but main goal is enabling modern large models to run on standard consumer GPUs without performance hits typically associated with aggressive memory swapping and needs for constant manual tweaks - New [documentation website](https://vladmandic.github.io/sdnext-docs/) with full search and tons of new documentation - New settings panel with simplified and streamlined configuration -We've also added support for several new models (see [supported models](https://vladmandic.github.io/sdnext-docs/Model-Support/) for full list) such as highly anticipated [NVLabs Sana](https://huggingface.co/Efficient-Large-Model/Sana_1600M_1024px) +We've also added support for several new models such as highly anticipated [NVLabs Sana](https://huggingface.co/Efficient-Large-Model/Sana_1600M_1024px) (see [supported models](https://vladmandic.github.io/sdnext-docs/Model-Support/) for full list) And several new SOTA video models: [Lightricks LTX-Video](https://huggingface.co/Lightricks/LTX-Video), [Hunyuan Video](https://huggingface.co/tencent/HunyuanVideo) and [Genmo Mochi.1 Preview](https://huggingface.co/genmo/mochi-1-preview) And a lot of **Control** and **IPAdapter** goodies - for **SDXL** there is new [ProMax](https://huggingface.co/xinsir/controlnet-union-sdxl-1.0), improved *Union* and *Tiling* models -- for **FLUX.1** there are [Flux Tools](https://blackforestlabs.ai/flux-1-tools/) as well as official *Canny* and *Depth* models and a cool [Redux](https://huggingface.co/black-forest-labs/FLUX.1-Redux-dev) model -- for **SD3.5** there are official *Canny*, *Blur* and *Depth* models in addition to existing 3rd party models as well as [InstantX](https://huggingface.co/InstantX/SD3.5-Large-IP-Adapter) IP-adapter +- for **FLUX.1** there are [Flux Tools](https://blackforestlabs.ai/flux-1-tools/) as well as official *Canny* and *Depth* models, + a cool [Redux](https://huggingface.co/black-forest-labs/FLUX.1-Redux-dev) model as well as [XLabs](https://huggingface.co/XLabs-AI/flux-ip-adapter-v2) IP-adapter +- for **SD3.5** there are official *Canny*, *Blur* and *Depth* models in addition to existing 3rd party models + as well as [InstantX](https://huggingface.co/InstantX/SD3.5-Large-IP-Adapter) IP-adapter Plus couple of new integrated workflows such as [FreeScale](https://github.com/ali-vilab/FreeScale) and [Style Aligned Image Generation](https://style-aligned-gen.github.io/) @@ -49,6 +52,10 @@ All-in-all, we're around ~160 commits worth of updates, check changelog for full *note*: when selecting tiles in control settings, you can also specify non-square ratios in which case it will use context-aware image resize to maintain overall composition *note*: available tiling options can be set in settings -> control +- **IP-Adapter** + - FLUX.1 [XLabs](https://huggingface.co/XLabs-AI/flux-ip-adapter-v2) v1 and v2 IP-adapter + - FLUX.1 secondary guidance, enabled using *Attention guidance* in advanced menu + - SD 3.5 [InstantX](https://huggingface.co/InstantX/SD3.5-Large-IP-Adapter) IP-adapter - [Flux Tools](https://blackforestlabs.ai/flux-1-tools/) **Redux** is actually a tool, **Fill** is inpaint/outpaint optimized version of *Flux-dev* **Canny** & **Depth** are optimized versions of *Flux-dev* for their respective tasks: they are *not* ControlNets that work on top of a model diff --git a/installer.py b/installer.py index d186cc5ac..6aa8bb060 100644 --- a/installer.py +++ b/installer.py @@ -459,7 +459,7 @@ def check_python(supported_minors=[9, 10, 11, 12], reason=None): def check_diffusers(): if args.skip_all or args.skip_git: return - sha = 'b64ca6c11cbc1644c22f1dae441c8124d588bb14' # diffusers commit hash + sha = '233dffdc3f56b26abaaba8363a5dd30dab7f0e40' # diffusers commit hash pkg = pkg_resources.working_set.by_key.get('diffusers', None) minor = int(pkg.version.split('.')[1] if pkg is not None else 0) cur = opts.get('diffusers_version', '') if minor > 0 else '' diff --git a/modules/errors.py b/modules/errors.py index 5f2c54cb7..6302057d7 100644 --- a/modules/errors.py +++ b/modules/errors.py @@ -36,7 +36,7 @@ def print_error_explanation(message): log.error(line) -def display(e: Exception, task, suppress=[]): +def display(e: Exception, task: str, suppress=[]): log.error(f"{task or 'error'}: {type(e).__name__}") console.print_exception(show_locals=False, max_frames=16, extra_lines=1, suppress=suppress, theme="ansi_dark", word_wrap=False, width=console.width) @@ -48,7 +48,7 @@ def display_once(e: Exception, task): already_displayed[task] = 1 -def run(code, task): +def run(code, task: str): try: code() except Exception as e: diff --git a/modules/ipadapter.py b/modules/ipadapter.py index c1b6ed52f..aa010c33d 100644 --- a/modules/ipadapter.py +++ b/modules/ipadapter.py @@ -11,12 +11,13 @@ from PIL import Image import diffusers import transformers -from modules import processing, shared, devices, sd_models +from modules import processing, shared, devices, sd_models, errors clip_loaded = None adapters_loaded = [] CLIP_ID = "h94/IP-Adapter" +OPEN_ID = "openai/clip-vit-large-patch14" SIGLIP_ID = 'google/siglip-so400m-patch14-384' ADAPTERS_NONE = { 'None': { 'name': 'none', 'repo': 'none', 'subfolder': 'none' }, @@ -136,14 +137,22 @@ def crop_images(images, crops): return images -def unapply(pipe): # pylint: disable=arguments-differ +def unapply(pipe, unload: bool = False): # pylint: disable=arguments-differ if len(adapters_loaded) == 0: return try: if hasattr(pipe, 'set_ip_adapter_scale'): pipe.set_ip_adapter_scale(0) - pipe.unload_ip_adapter() - if hasattr(pipe, 'unet') and hasattr(pipe.unet, 'config') and pipe.unet.config.encoder_hid_dim_type == 'ip_image_proj': + if unload: + shared.log.debug('IP adapter unload') + pipe.unload_ip_adapter() + if hasattr(pipe, 'unet'): + module = pipe.unet + elif hasattr(pipe, 'transformer'): + module = pipe.transformer + else: + module = None + if module is not None and hasattr(module, 'config') and module.config.encoder_hid_dim_type == 'ip_image_proj': pipe.unet.encoder_hid_proj = None pipe.config.encoder_hid_dim_type = None pipe.unet.set_default_attn_processor() @@ -171,8 +180,8 @@ def load_image_encoder(pipe: diffusers.DiffusionPipeline, adapter_names: list[st clip_repo = SIGLIP_ID clip_subfolder = None elif shared.sd_model_type == 'f1': - shared.log.error(f'IP adapter: adapter={adapter_name} type={shared.sd_model_type} cls={shared.sd_model.__class__.__name__}: unsupported base model') - return False + clip_repo = OPEN_ID + clip_subfolder = None else: shared.log.error(f'IP adapter: unknown model type: {adapter_name}') return False @@ -181,13 +190,22 @@ def load_image_encoder(pipe: diffusers.DiffusionPipeline, adapter_names: list[st if pipe.image_encoder is None or clip_loaded != f'{clip_repo}/{clip_subfolder}': try: if shared.sd_model_type == 'sd3': - pipe.image_encoder = transformers.SiglipVisionModel.from_pretrained(clip_repo, torch_dtype=devices.dtype, cache_dir=shared.opts.hfcache_dir) + image_encoder = transformers.SiglipVisionModel.from_pretrained(clip_repo, torch_dtype=devices.dtype, cache_dir=shared.opts.hfcache_dir) else: - pipe.image_encoder = transformers.CLIPVisionModelWithProjection.from_pretrained(clip_repo, subfolder=clip_subfolder, torch_dtype=devices.dtype, cache_dir=shared.opts.hfcache_dir, use_safetensors=True) - shared.log.debug(f'IP adapter load: encoder="{clip_repo}/{clip_subfolder}" cls={pipe.image_encoder.__class__.__name__}') + if clip_subfolder is None: + image_encoder = transformers.CLIPVisionModelWithProjection.from_pretrained(clip_repo, torch_dtype=devices.dtype, cache_dir=shared.opts.hfcache_dir, use_safetensors=True) + shared.log.debug(f'IP adapter load: encoder="{clip_repo}" cls={pipe.image_encoder.__class__.__name__}') + else: + image_encoder = transformers.CLIPVisionModelWithProjection.from_pretrained(clip_repo, subfolder=clip_subfolder, torch_dtype=devices.dtype, cache_dir=shared.opts.hfcache_dir, use_safetensors=True) + shared.log.debug(f'IP adapter load: encoder="{clip_repo}/{clip_subfolder}" cls={pipe.image_encoder.__class__.__name__}') + if hasattr(pipe, 'register_modules'): + pipe.register_modules(image_encoder=image_encoder) + else: + pipe.image_encoder = image_encoder clip_loaded = f'{clip_repo}/{clip_subfolder}' except Exception as e: shared.log.error(f'IP adapter load: encoder="{clip_repo}/{clip_subfolder}" {e}') + errors.display(e, 'IP adapter: type=encoder') return False sd_models.move_model(pipe.image_encoder, devices.device) return True @@ -198,12 +216,17 @@ def load_feature_extractor(pipe): if pipe.feature_extractor is None: try: if shared.sd_model_type == 'sd3': - pipe.feature_extractor = transformers.SiglipImageProcessor.from_pretrained(SIGLIP_ID, torch_dtype=devices.dtype, cache_dir=shared.opts.hfcache_dir) + feature_extractor = transformers.SiglipImageProcessor.from_pretrained(SIGLIP_ID, torch_dtype=devices.dtype, cache_dir=shared.opts.hfcache_dir) + else: + feature_extractor = transformers.CLIPImageProcessor() + if hasattr(pipe, 'register_modules'): + pipe.register_modules(feature_extractor=feature_extractor) else: - pipe.feature_extractor = transformers.CLIPImageProcessor() + pipe.feature_extractor = feature_extractor shared.log.debug(f'IP adapter load: extractor={pipe.feature_extractor.__class__.__name__}') except Exception as e: shared.log.error(f'IP adapter load: extractor {e}') + errors.display(e, 'IP adapter: type=extractor') return False return True @@ -266,7 +289,7 @@ def apply(pipe, p: processing.StableDiffusionProcessing, adapter_names=[], adapt adapters = [ADAPTERS.get(adapter_name, None) for adapter_name in adapter_names if adapter_name.lower() != 'none'] if len(adapters) == 0: - unapply(pipe) + unapply(pipe, getattr(p, 'ip_adapter_unload', False)) if hasattr(p, 'ip_adapter_images'): del p.ip_adapter_images return False @@ -286,7 +309,7 @@ def apply(pipe, p: processing.StableDiffusionProcessing, adapter_names=[], adapt shared.log.error('IP adapter: no image provided') adapters = [] # unload adapter if previously loaded as it will cause runtime errors if len(adapters) == 0: - unapply(pipe) + unapply(pipe, getattr(p, 'ip_adapter_unload', False)) if hasattr(p, 'ip_adapter_images'): del p.ip_adapter_images return False @@ -335,4 +358,5 @@ def apply(pipe, p: processing.StableDiffusionProcessing, adapter_names=[], adapt shared.log.info(f'IP adapter: {ip_str} image={adapter_images} mask={adapter_masks is not None} time={t1-t0:.2f}') except Exception as e: shared.log.error(f'IP adapter load: adapters={adapter_names} repo={repos} folders={subfolders} names={names} {e}') + errors.display(e, 'IP adapter: type=adapter') return True diff --git a/modules/pag/__init__.py b/modules/pag/__init__.py index 8fe54c198..b7a56c40d 100644 --- a/modules/pag/__init__.py +++ b/modules/pag/__init__.py @@ -17,17 +17,22 @@ def apply(p: processing.StableDiffusionProcessing): # pylint: disable=arguments- unapply() if p.pag_scale == 0: return - if sd_models.get_diffusers_task(shared.sd_model) != sd_models.DiffusersTaskType.TEXT_2_IMAGE: - shared.log.warning(f'PAG: pipeline={c} not implemented') - return None if 'PAG' in shared.sd_model.__class__.__name__: pass elif detect.is_sd15(c): + if sd_models.get_diffusers_task(shared.sd_model) != sd_models.DiffusersTaskType.TEXT_2_IMAGE: + shared.log.warning(f'PAG: pipeline={c} not implemented') + return None orig_pipeline = shared.sd_model shared.sd_model = sd_models.switch_pipe(StableDiffusionPAGPipeline, shared.sd_model) elif detect.is_sdxl(c): + if sd_models.get_diffusers_task(shared.sd_model) != sd_models.DiffusersTaskType.TEXT_2_IMAGE: + shared.log.warning(f'PAG: pipeline={c} not implemented') + return None orig_pipeline = shared.sd_model shared.sd_model = sd_models.switch_pipe(StableDiffusionXLPAGPipeline, shared.sd_model) + elif detect.is_f1(c): + p.task_args['true_cfg_scale'] = p.pag_scale else: shared.log.warning(f'PAG: pipeline={c} required={StableDiffusionPipeline.__name__}') return None diff --git a/modules/processing.py b/modules/processing.py index f39393f95..6d9b64c17 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -451,7 +451,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: if shared.native: from modules import ipadapter - ipadapter.unapply(shared.sd_model) + ipadapter.unapply(shared.sd_model, unload=getattr(p, 'ip_adapter_unload', False)) if shared.opts.include_mask: if shared.opts.mask_apply_overlay and p.overlay_images is not None and len(p.overlay_images): diff --git a/scripts/ipadapter.py b/scripts/ipadapter.py index 60c70b9dc..5ca4ca578 100644 --- a/scripts/ipadapter.py +++ b/scripts/ipadapter.py @@ -57,6 +57,7 @@ def ui(self, _is_img2img): mask_galleries = [] with gr.Row(): num_adapters = gr.Slider(label="Active IP adapters", minimum=1, maximum=MAX_ADAPTERS, step=1, value=1, scale=1) + unload_adapter = gr.Checkbox(label='Unload adapter', value=False, interactive=True) for i in range(MAX_ADAPTERS): with gr.Accordion(f'Adapter {i+1}', visible=i==0) as unit: with gr.Row(): @@ -85,7 +86,7 @@ def ui(self, _is_img2img): layers_label = gr.HTML('InstantStyle: advanced layer activation', visible=False) layers = gr.Text(label='Layer scales', placeholder='{\n"down": {"block_2": [0.0, 1.0]},\n"up": {"block_0": [0.0, 1.0, 0.0]}\n}', rows=1, type='text', interactive=True, lines=5, visible=False, show_label=False) layers_active.change(fn=self.display_advanced, inputs=[layers_active], outputs=[layers_label, layers]) - return [num_adapters] + adapters + scales + files + crops + starts + ends + masks + [layers_active] + [layers] + return [num_adapters] + [unload_adapter] + adapters + scales + files + crops + starts + ends + masks + [layers_active] + [layers] def process(self, p: processing.StableDiffusionProcessing, *args): # pylint: disable=arguments-differ if not shared.native: @@ -94,6 +95,7 @@ def process(self, p: processing.StableDiffusionProcessing, *args): # pylint: dis if len(args) == 0: return units = args.pop(0) + unload = args.pop(0) if getattr(p, 'ip_adapter_names', []) == []: p.ip_adapter_names = args[:MAX_ADAPTERS][:units] if getattr(p, 'ip_adapter_scales', [0.0]) == [0.0]: @@ -110,6 +112,7 @@ def process(self, p: processing.StableDiffusionProcessing, *args): # pylint: dis p.ip_adapter_masks = args[MAX_ADAPTERS*6:MAX_ADAPTERS*7][:units] p.ip_adapter_masks = [x for x in p.ip_adapter_masks if x] layers_active, layers = args[MAX_ADAPTERS*7:MAX_ADAPTERS*8] + p.ip_adapter_unload = unload if layers_active and len(layers) > 0: try: layers = json.loads(layers) From ef0b90b126693c1f20c40771824ca5926f937599 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sun, 22 Dec 2024 03:04:40 +0300 Subject: [PATCH 131/249] IPEX disable interpolate cpu fallback with IPEX 2.5 --- installer.py | 2 ++ modules/intel/ipex/hijacks.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/installer.py b/installer.py index 6aa8bb060..07917986f 100644 --- a/installer.py +++ b/installer.py @@ -637,6 +637,8 @@ def install_ipex(torch_command): os.environ.setdefault('NEOReadDebugKeys', '1') if os.environ.get("ClDeviceGlobalMemSizeAvailablePercent", None) is None: os.environ.setdefault('ClDeviceGlobalMemSizeAvailablePercent', '100') + if os.environ.get("PYTORCH_ENABLE_XPU_FALLBACK", None) is None: + os.environ.setdefault('PYTORCH_ENABLE_XPU_FALLBACK', '1') if "linux" in sys.platform: torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.5.1+cxx11.abi torchvision==0.20.1+cxx11.abi intel-extension-for-pytorch==2.5.10+xpu oneccl_bind_pt==2.5.0+xpu --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/cn/') # torch_command = os.environ.get('TORCH_COMMAND', 'torch torchvision --index-url https://download.pytorch.org/whl/test/xpu') # test wheels are stable previews, significantly slower than IPEX diff --git a/modules/intel/ipex/hijacks.py b/modules/intel/ipex/hijacks.py index 43721a64c..b1c9a1182 100644 --- a/modules/intel/ipex/hijacks.py +++ b/modules/intel/ipex/hijacks.py @@ -313,7 +313,7 @@ def torch_load(f, map_location=None, *args, **kwargs): # Hijack Functions: def ipex_hijacks(legacy=True): - if legacy: + if legacy and float(torch.__version__[:3]) < 2.5: torch.nn.functional.interpolate = interpolate torch.tensor = torch_tensor torch.Tensor.to = Tensor_to From 312e478fccf0ed36e651ea203c2ad5c7921f4bcc Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 22 Dec 2024 08:20:16 -0500 Subject: [PATCH 132/249] fix wildcards Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 7 ++++--- modules/processing_vae.py | 3 ++- modules/styles.py | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 90eda3736..83eda417d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,8 @@ # Change Log for SD.Next -## Update for 2024-12-21 +## Update for 2024-12-22 -### Highlights for 2024-12-21 +### Highlights for 2024-12-22 ### SD.Next Xmass edition: *What's new?* @@ -32,7 +32,7 @@ All-in-all, we're around ~160 commits worth of updates, check changelog for full [ReadMe](https://github.com/vladmandic/automatic/blob/master/README.md) | [ChangeLog](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) | [Docs](https://vladmandic.github.io/sdnext-docs/) | [WiKi](https://github.com/vladmandic/automatic/wiki) | [Discord](https://discord.com/invite/sd-next-federal-batch-inspectors-1101998836328697867) -## Details for 2024-12-21 +## Details for 2024-12-22 ### New models and integrations @@ -217,6 +217,7 @@ All-in-all, we're around ~160 commits worth of updates, check changelog for full - fix xyz-grid with lora none - fix svd image2video - fix gallery display during generate +- fix wildcards replacement to be unique ## Update for 2024-11-21 diff --git a/modules/processing_vae.py b/modules/processing_vae.py index 772a48adc..5d8f9fc84 100644 --- a/modules/processing_vae.py +++ b/modules/processing_vae.py @@ -144,7 +144,8 @@ def full_vae_decode(latents, model): decoded = model.vae.decode(latents, return_dict=False)[0] except Exception as e: shared.log.error(f'VAE decode: {stats} {e}') - errors.display(e, 'VAE decode') + if 'out of memory' not in str(e): + errors.display(e, 'VAE decode') decoded = [] if hasattr(model.vae, "orig_dtype"): diff --git a/modules/styles.py b/modules/styles.py index 0dd48eb7f..d0228d33a 100644 --- a/modules/styles.py +++ b/modules/styles.py @@ -52,7 +52,7 @@ def check_files(prompt, wildcard, files): choice = random.choice(lines).strip(' \n') if '|' in choice: choice = random.choice(choice.split('|')).strip(' []{}\n') - prompt = prompt.replace(f"__{wildcard}__", choice) + prompt = prompt.replace(f"__{wildcard}__", choice, 1) shared.log.debug(f'Wildcards apply: wildcard="{wildcard}" choice="{choice}" file="{file}" choices={len(lines)}') replaced.append(wildcard) return prompt, True From e99371328c000ade3e2ca8f03f3d0dada8247ca9 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 22 Dec 2024 08:39:31 -0500 Subject: [PATCH 133/249] update all todo statements Signed-off-by: Vladimir Mandic --- TODO.md | 28 +++++++++---------- installer.py | 4 +-- modules/api/control.py | 2 +- modules/consistory/consistory_unet_sdxl.py | 1 - modules/control/run.py | 2 +- modules/ctrlx/__init__.py | 10 +++---- modules/face/instantid_model.py | 1 - modules/face/photomaker_model.py | 10 +++---- modules/ggml/gguf_tensor.py | 1 - modules/hidiffusion/hidiffusion.py | 6 ++-- modules/images_resize.py | 2 +- modules/instantir/aggregator.py | 1 - .../ip_adapter/attention_processor.py | 11 -------- modules/lora/lora_extract.py | 2 +- modules/lora/networks.py | 2 +- modules/meissonic/pipeline_img2img.py | 3 -- modules/meissonic/pipeline_inpaint.py | 3 -- modules/meissonic/transformer.py | 5 ---- modules/model_flux.py | 4 +-- modules/model_sd3.py | 2 +- modules/omnigen/utils.py | 1 - modules/onnx_impl/pipelines/__init__.py | 2 +- modules/pag/pipe_sd.py | 2 -- modules/pag/pipe_sdxl.py | 2 -- modules/processing_class.py | 2 +- modules/processing_diffusers.py | 7 ----- modules/pulid/eva_clip/hf_model.py | 3 -- modules/schedulers/scheduler_tcd.py | 1 - modules/schedulers/scheduler_ufogen.py | 5 ---- modules/schedulers/scheduler_vdm.py | 2 +- modules/sd_hijack_hypertile.py | 2 +- modules/sd_models.py | 2 +- modules/sd_samplers_common.py | 10 ------- modules/sd_unet.py | 2 +- modules/segmoe/segmoe_model.py | 4 +-- modules/xadapter/adapter.py | 2 -- modules/xadapter/unet_adapter.py | 6 ++-- scripts/instantir.py | 2 +- scripts/stablevideodiffusion.py | 2 +- 39 files changed, 46 insertions(+), 113 deletions(-) diff --git a/TODO.md b/TODO.md index ad43e6d06..bf0cecbb4 100644 --- a/TODO.md +++ b/TODO.md @@ -10,9 +10,7 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma ## Future Candidates -- Flux IPAdapter: -- Flux NF4: -- GGUF: +- Flux NF4 loader: ## Other @@ -22,15 +20,15 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma ## Code TODO -- python 3.12.4 or higher cause a mess with pydantic -- enable ROCm for windows when available -- enable full VAE mode for resize-latent -- remove duplicate mask params -- fix flux loader for civitai nf4 models -- implement model in-memory caching -- hypertile vae breaks for diffusers when using non-standard sizes -- forcing reloading entire model as loading transformers only leads to massive memory usage -- lora-direct with bnb -- make lora for quantized flux -- control script process -- monkey-patch for modernui missing tabs.select event +- TODO install: python 3.12.4 or higher cause a mess with pydantic (fixme) +- TODO install: enable ROCm for windows when available (fixme) +- TODO resize image: enable full VAE mode for resize-latent (fixme) +- TODO processing: remove duplicate mask params (fixme) +- TODO flux: fix loader for civitai nf4 models (fixme) +- TODO model loader: implement model in-memory caching (fixme) +- TODO hypertile: vae breaks when using non-standard sizes (fixme) +- TODO model load: force-reloading entire model as loading transformers only leads to massive memory usage (fixme) +- TODO lora load: direct with bnb (fixme) +- TODO: lora make: support quantized flux (fixme) +- TODO control: support scripts via api (fixme) +- TODO modernui: monkey-patch for missing tabs.select event (fixme) diff --git a/installer.py b/installer.py index 07917986f..55f23c27f 100644 --- a/installer.py +++ b/installer.py @@ -430,7 +430,7 @@ def check_python(supported_minors=[9, 10, 11, 12], reason=None): if args.quick: return log.info(f'Python: version={platform.python_version()} platform={platform.system()} bin="{sys.executable}" venv="{sys.prefix}"') - if int(sys.version_info.major) == 3 and int(sys.version_info.minor) == 12 and int(sys.version_info.micro) > 3: # TODO python 3.12.4 or higher cause a mess with pydantic + if int(sys.version_info.major) == 3 and int(sys.version_info.minor) == 12 and int(sys.version_info.micro) > 3: # TODO install: python 3.12.4 or higher cause a mess with pydantic log.error(f"Python version incompatible: {sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro} required 3.12.3 or lower") if reason is not None: log.error(reason) @@ -552,7 +552,7 @@ def install_rocm_zluda(): log.info(msg) torch_command = '' if sys.platform == "win32": - # TODO enable ROCm for windows when available + # TODO install: enable ROCm for windows when available if args.device_id is not None: if os.environ.get('HIP_VISIBLE_DEVICES', None) is not None: diff --git a/modules/api/control.py b/modules/api/control.py index 345930341..411f71ff8 100644 --- a/modules/api/control.py +++ b/modules/api/control.py @@ -159,7 +159,7 @@ def post_control(self, req: ReqControl): output_images = [] output_processed = [] output_info = '' - # TODO control script process + # TODO control: support scripts via api # init script args, call scripts.script_control.run, call scripts.script_control.after run.control_set({ 'do_not_save_grid': not req.save_images, 'do_not_save_samples': not req.save_images, **self.prepare_ip_adapter(req) }) run.control_set(getattr(req, "extra", {})) diff --git a/modules/consistory/consistory_unet_sdxl.py b/modules/consistory/consistory_unet_sdxl.py index 4dd9b42d2..940b4ba01 100644 --- a/modules/consistory/consistory_unet_sdxl.py +++ b/modules/consistory/consistory_unet_sdxl.py @@ -916,7 +916,6 @@ def forward( # 1. time timesteps = timestep if not torch.is_tensor(timesteps): - # TODO: this requires sync between CPU and GPU. So try to pass timesteps as tensors if you can # This would be a good case for the `match` statement (Python 3.10+) is_mps = sample.device.type == "mps" if isinstance(timestep, float): diff --git a/modules/control/run.py b/modules/control/run.py index efe9c0b8b..8cecb93af 100644 --- a/modules/control/run.py +++ b/modules/control/run.py @@ -311,7 +311,7 @@ def control_run(state: str = '', # processing.process_init(p) resize_mode_before = resize_mode_before if resize_name_before != 'None' and inputs is not None and len(inputs) > 0 else 0 - # TODO monkey-patch for modernui missing tabs.select event + # TODO modernui: monkey-patch for missing tabs.select event if selected_scale_tab_before == 0 and resize_name_before != 'None' and scale_by_before != 1 and inputs is not None and len(inputs) > 0: shared.log.debug('Control: override resize mode=before') selected_scale_tab_before = 1 diff --git a/modules/ctrlx/__init__.py b/modules/ctrlx/__init__.py index 87ff52f6c..07d06aeac 100644 --- a/modules/ctrlx/__init__.py +++ b/modules/ctrlx/__init__.py @@ -136,7 +136,7 @@ def appearance_guidance_scale(self): @torch.no_grad() def __call__( self, - prompt: Union[str, List[str]] = None, # TODO: Support prompt_2 and negative_prompt_2 + prompt: Union[str, List[str]] = None, structure_prompt: Optional[Union[str, List[str]]] = None, appearance_prompt: Optional[Union[str, List[str]]] = None, structure_image: Optional[PipelineImageInput] = None, @@ -180,7 +180,6 @@ def __call__( callback_on_step_end_tensor_inputs: List[str] = ["latents"], **kwargs, ): - # TODO: Add function argument documentation callback = kwargs.pop("callback", None) callback_steps = kwargs.pop("callback_steps", None) @@ -205,7 +204,7 @@ def __call__( target_size = target_size or (height, width) # 1. Check inputs. Raise error if not correct - self.check_inputs( # TODO: Custom check_inputs for our method + self.check_inputs( prompt, None, # prompt_2 height, @@ -425,7 +424,7 @@ def denoising_value_valid(dnv): # 7.2 Optionally get guidance scale embedding timestep_cond = None - if self.unet.config.time_cond_proj_dim is not None: # TODO: Make guidance scale embedding work with batch_order + if self.unet.config.time_cond_proj_dim is not None: guidance_scale_tensor = torch.tensor(self.guidance_scale - 1).repeat(batch_size * num_images_per_prompt) timestep_cond = self.get_guidance_scale_embedding( guidance_scale_tensor, embedding_dim=self.unet.config.time_cond_proj_dim @@ -457,7 +456,6 @@ def denoising_value_valid(dnv): register_attr(self, t=t.item(), do_control=True, batch_order=batch_order) - # TODO: For now, assume we are doing classifier-free guidance, support no CF-guidance later latent_model_input = self.scheduler.scale_model_input(latents, t) structure_latent_model_input = self.scheduler.scale_model_input(structure_latents, t) appearance_latent_model_input = self.scheduler.scale_model_input(appearance_latents, t) @@ -563,7 +561,7 @@ def denoising_value_valid(dnv): # Self-recurrence for _ in range(self_recurrence_schedule[i]): if hasattr(self.scheduler, "_step_index"): # For fancier schedulers - self.scheduler._step_index -= 1 # TODO: Does this actually work? + self.scheduler._step_index -= 1 t_prev = 0 if i + 1 >= num_inference_steps else timesteps[i + 1] latents = noise_t2t(self.scheduler, t_prev, t, latents) diff --git a/modules/face/instantid_model.py b/modules/face/instantid_model.py index 51a4d7850..543b39ded 100644 --- a/modules/face/instantid_model.py +++ b/modules/face/instantid_model.py @@ -344,7 +344,6 @@ def __call__( return hidden_states def _memory_efficient_attention_xformers(self, query, key, value, attention_mask): - # TODO attention_mask query = query.contiguous() key = key.contiguous() value = value.contiguous() diff --git a/modules/face/photomaker_model.py b/modules/face/photomaker_model.py index b62fe73b8..3595c6a36 100644 --- a/modules/face/photomaker_model.py +++ b/modules/face/photomaker_model.py @@ -244,7 +244,7 @@ def encode_prompt_with_trigger_word( prompt_embeds_list = [] prompts = [prompt, prompt_2] for prompt, tokenizer, text_encoder in zip(prompts, tokenizers, text_encoders): - input_ids = tokenizer.encode(prompt) # TODO: batch encode + input_ids = tokenizer.encode(prompt) clean_index = 0 clean_input_ids = [] class_token_index = [] @@ -296,7 +296,7 @@ def encode_prompt_with_trigger_word( prompt_embeds = torch.concat(prompt_embeds_list, dim=-1) prompt_embeds = prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device) - class_tokens_mask = class_tokens_mask.to(device=device) # TODO: ignoring two-prompt case + class_tokens_mask = class_tokens_mask.to(device=device) return prompt_embeds, pooled_prompt_embeds, class_tokens_mask @@ -332,7 +332,7 @@ def __call__( callback_steps: int = 1, # Added parameters (for PhotoMaker) input_id_images: PipelineImageInput = None, - start_merge_step: int = 0, # TODO: change to `style_strength_ratio` in the future + start_merge_step: int = 0, class_tokens_mask: Optional[torch.LongTensor] = None, prompt_embeds_text_only: Optional[torch.FloatTensor] = None, pooled_prompt_embeds_text_only: Optional[torch.FloatTensor] = None, @@ -410,7 +410,7 @@ def __call__( ( prompt_embeds_text_only, negative_prompt_embeds, - pooled_prompt_embeds_text_only, # TODO: replace the pooled_prompt_embeds with text only prompt + pooled_prompt_embeds_text_only, negative_pooled_prompt_embeds, ) = self.encode_prompt( prompt=prompt_text_only, @@ -431,7 +431,7 @@ def __call__( if not isinstance(input_id_images[0], torch.Tensor): id_pixel_values = self.id_image_processor(input_id_images, return_tensors="pt").pixel_values - id_pixel_values = id_pixel_values.unsqueeze(0).to(device=device, dtype=dtype) # TODO: multiple prompts + id_pixel_values = id_pixel_values.unsqueeze(0).to(device=device, dtype=dtype) # 6. Get the update text embedding with the stacked ID embedding prompt_embeds = self.id_encoder(id_pixel_values, prompt_embeds, class_tokens_mask) diff --git a/modules/ggml/gguf_tensor.py b/modules/ggml/gguf_tensor.py index 4bc9117cb..8b2f608ac 100644 --- a/modules/ggml/gguf_tensor.py +++ b/modules/ggml/gguf_tensor.py @@ -131,7 +131,6 @@ def get_dequantized_tensor(self): if self._ggml_quantization_type in TORCH_COMPATIBLE_QTYPES: return self.quantized_data.to(self.compute_dtype) elif self._ggml_quantization_type in DEQUANTIZE_FUNCTIONS: - # TODO(ryand): Look into how the dtype param is intended to be used. return dequantize( data=self.quantized_data, qtype=self._ggml_quantization_type, oshape=self.tensor_shape, dtype=None ).to(self.compute_dtype) diff --git a/modules/hidiffusion/hidiffusion.py b/modules/hidiffusion/hidiffusion.py index 7874f03af..d6f68eb15 100644 --- a/modules/hidiffusion/hidiffusion.py +++ b/modules/hidiffusion/hidiffusion.py @@ -234,7 +234,7 @@ def window_reverse(windows, window_size, H, W, shift_size): norm_hidden_states = self.norm2(hidden_states) norm_hidden_states = norm_hidden_states * (1 + scale_mlp) + shift_mlp if self._chunk_size is not None: - ff_output = _chunked_feed_forward(self.ff, norm_hidden_states, self._chunk_dim, self._chunk_size) # pylint: disable=undefined-variable # TODO hidiffusion undefined + ff_output = _chunked_feed_forward(self.ff, norm_hidden_states, self._chunk_dim, self._chunk_size) # pylint: disable=undefined-variable else: ff_output = self.ff(norm_hidden_states) if self.use_ada_layer_norm_zero: @@ -308,7 +308,7 @@ def forward( self.T1 = int(self.max_timestep * self.T1_ratio) output_states = () - _scale = cross_attention_kwargs.get("scale", 1.0) if cross_attention_kwargs is not None else 1.0 # TODO hidiffusion unused + _scale = cross_attention_kwargs.get("scale", 1.0) if cross_attention_kwargs is not None else 1.0 blocks = list(zip(self.resnets, self.attentions)) @@ -407,7 +407,7 @@ def forward( encoder_attention_mask: Optional[torch.FloatTensor] = None, ) -> torch.FloatTensor: - def fix_scale(first, second): # TODO hidiffusion breaks hidden_scale.shape on 3rd generate with sdxl + def fix_scale(first, second): if (first.shape[-1] != second.shape[-1] or first.shape[-2] != second.shape[-2]): rescale = min(second.shape[-2] / first.shape[-2], second.shape[-1] / first.shape[-1]) # log.debug(f"HiDiffusion rescale: {hidden_states.shape} => {res_hidden_states_tuple[0].shape} scale={rescale}") diff --git a/modules/images_resize.py b/modules/images_resize.py index 5b7c816f8..362be79ee 100644 --- a/modules/images_resize.py +++ b/modules/images_resize.py @@ -11,7 +11,7 @@ def resize_image(resize_mode: int, im: Image.Image, width: int, height: int, ups def latent(im, w, h, upscaler): from modules.processing_vae import vae_encode, vae_decode import torch - latents = vae_encode(im, shared.sd_model, full_quality=False) # TODO enable full VAE mode for resize-latent + latents = vae_encode(im, shared.sd_model, full_quality=False) # TODO resize image: enable full VAE mode for resize-latent latents = torch.nn.functional.interpolate(latents, size=(int(h // 8), int(w // 8)), mode=upscaler["mode"], antialias=upscaler["antialias"]) im = vae_decode(latents, shared.sd_model, output_type='pil', full_quality=False)[0] return im diff --git a/modules/instantir/aggregator.py b/modules/instantir/aggregator.py index fd6151003..1950f9efa 100644 --- a/modules/instantir/aggregator.py +++ b/modules/instantir/aggregator.py @@ -823,7 +823,6 @@ def forward( # 1. time timesteps = timestep if not torch.is_tensor(timesteps): - # TODO: this requires sync between CPU and GPU. So try to pass timesteps as tensors if you can # This would be a good case for the `match` statement (Python 3.10+) is_mps = sample.device.type == "mps" if isinstance(timestep, float): diff --git a/modules/instantir/ip_adapter/attention_processor.py b/modules/instantir/ip_adapter/attention_processor.py index ed6cf755f..68dce4281 100644 --- a/modules/instantir/ip_adapter/attention_processor.py +++ b/modules/instantir/ip_adapter/attention_processor.py @@ -390,7 +390,6 @@ def __call__( value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) # the output of sdp = (batch, num_heads, seq_len, head_dim) - # TODO: add support for attn.scale when we move to Torch 2.1 hidden_states = F.scaled_dot_product_attention( query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False ) @@ -496,7 +495,6 @@ def __call__( value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) # the output of sdp = (batch, num_heads, seq_len, head_dim) - # TODO: add support for attn.scale when we move to Torch 2.1 hidden_states = F.scaled_dot_product_attention( query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False ) @@ -640,7 +638,6 @@ def __call__( value_1 = value_1.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) # the output of sdp = (batch, num_heads, seq_len, head_dim) - # TODO: add support for attn.scale when we move to Torch 2.1 hidden_states_0 = F.scaled_dot_product_attention( query_0, key_0, value_0, attn_mask=attention_mask, dropout_p=0.0, is_causal=False ) @@ -654,7 +651,6 @@ def __call__( ) hidden_states_0 = hidden_states_0 + ref_scale * _hidden_states_0 * 10 - # TODO: drop this cross-attn _hidden_states_1 = F.scaled_dot_product_attention( query_1, key_0, value_0, attn_mask=attention_mask, dropout_p=0.0, is_causal=False ) @@ -771,7 +767,6 @@ def __call__( value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) # the output of sdp = (batch, num_heads, seq_len, head_dim) - # TODO: add support for attn.scale when we move to Torch 2.1 hidden_states = F.scaled_dot_product_attention( query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False ) @@ -883,7 +878,6 @@ def __call__( value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) # the output of sdp = (batch, num_heads, seq_len, head_dim) - # TODO: add support for attn.scale when we move to Torch 2.1 hidden_states = F.scaled_dot_product_attention( query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False ) @@ -1018,7 +1012,6 @@ def __call__( value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) # the output of sdp = (batch, num_heads, seq_len, head_dim) - # TODO: add support for attn.scale when we move to Torch 2.1 hidden_states = F.scaled_dot_product_attention( query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False ) @@ -1034,7 +1027,6 @@ def __call__( ip_value = ip_value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) # the output of sdp = (batch, num_heads, seq_len, head_dim) - # TODO: add support for attn.scale when we move to Torch 2.1 ip_hidden_states = F.scaled_dot_product_attention( query, ip_key, ip_value, attn_mask=None, dropout_p=0.0, is_causal=False ) @@ -1161,7 +1153,6 @@ def __call__( value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) # the output of sdp = (batch, num_heads, seq_len, head_dim) - # TODO: add support for attn.scale when we move to Torch 2.1 hidden_states = F.scaled_dot_product_attention( query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False ) @@ -1181,7 +1172,6 @@ def __call__( ip_value = ip_value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) # the output of sdp = (batch, num_heads, seq_len, head_dim) - # TODO: add support for attn.scale when we move to Torch 2.1 ip_hidden_states = F.scaled_dot_product_attention( query, ip_key, ip_value, attn_mask=None, dropout_p=0.0, is_causal=False ) @@ -1337,7 +1327,6 @@ def __call__( value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) # the output of sdp = (batch, num_heads, seq_len, head_dim) - # TODO: add support for attn.scale when we move to Torch 2.1 hidden_states = F.scaled_dot_product_attention( query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False ) diff --git a/modules/lora/lora_extract.py b/modules/lora/lora_extract.py index c7deb2530..58cd065bb 100644 --- a/modules/lora/lora_extract.py +++ b/modules/lora/lora_extract.py @@ -182,7 +182,7 @@ def make_lora(fn, maxrank, auto_rank, rank_ratio, modules, overwrite): progress.remove_task(task) t3 = time.time() - # TODO: make lora for quantized flux + # TODO: lora make support quantized flux # if 'te' in modules and getattr(shared.sd_model, 'transformer', None) is not None: # for name, module in shared.sd_model.transformer.named_modules(): # if "norm" in name and "linear" not in name: diff --git a/modules/lora/networks.py b/modules/lora/networks.py index df7778ead..fc90ddd2d 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -416,7 +416,7 @@ def network_apply_direct(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn. if deactivate: updown *= -1 if getattr(self, "quant_type", None) in ['nf4', 'fp4'] and bnb is not None: - try: # TODO lora-direct with bnb + try: # TODO lora load: direct with bnb weight = bnb.functional.dequantize_4bit(self.weight, quant_state=self.quant_state, quant_type=self.quant_type, blocksize=self.blocksize) new_weight = weight.to(devices.device) + updown.to(devices.device) self.weight = bnb.nn.Params4bit(new_weight, quant_state=self.quant_state, quant_type=self.quant_type, blocksize=self.blocksize) diff --git a/modules/meissonic/pipeline_img2img.py b/modules/meissonic/pipeline_img2img.py index f26af123d..13e5c3717 100644 --- a/modules/meissonic/pipeline_img2img.py +++ b/modules/meissonic/pipeline_img2img.py @@ -56,9 +56,6 @@ class Img2ImgPipeline(DiffusionPipeline): model_cpu_offload_seq = "text_encoder->transformer->vqvae" - # TODO - when calling self.vqvae.quantize, it uses self.vqvae.quantize.embedding.weight before - # the forward method of self.vqvae.quantize, so the hook doesn't get called to move the parameter - # off the meta device. There should be a way to fix this instead of just not offloading it _exclude_from_cpu_offload = ["vqvae"] def __init__( diff --git a/modules/meissonic/pipeline_inpaint.py b/modules/meissonic/pipeline_inpaint.py index 994846fba..d405afa53 100644 --- a/modules/meissonic/pipeline_inpaint.py +++ b/modules/meissonic/pipeline_inpaint.py @@ -53,9 +53,6 @@ class InpaintPipeline(DiffusionPipeline): model_cpu_offload_seq = "text_encoder->transformer->vqvae" - # TODO - when calling self.vqvae.quantize, it uses self.vqvae.quantize.embedding.weight before - # the forward method of self.vqvae.quantize, so the hook doesn't get called to move the parameter - # off the meta device. There should be a way to fix this instead of just not offloading it _exclude_from_cpu_offload = ["vqvae"] def __init__( diff --git a/modules/meissonic/transformer.py b/modules/meissonic/transformer.py index 64f91baa2..43e77ddc7 100644 --- a/modules/meissonic/transformer.py +++ b/modules/meissonic/transformer.py @@ -341,11 +341,6 @@ def __call__( key = torch.cat([encoder_hidden_states_key_proj, key], dim=2) value = torch.cat([encoder_hidden_states_value_proj, value], dim=2) - # if image_rotary_emb is not None: # TODO broken import - # from .embeddings import apply_rotary_emb - # query = apply_rotary_emb(query, image_rotary_emb) - # key = apply_rotary_emb(key, image_rotary_emb) - hidden_states = F.scaled_dot_product_attention(query, key, value, dropout_p=0.0, is_causal=False) hidden_states = hidden_states.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim) hidden_states = hidden_states.to(query.dtype) diff --git a/modules/model_flux.py b/modules/model_flux.py index 0a13a4d46..f2286866e 100644 --- a/modules/model_flux.py +++ b/modules/model_flux.py @@ -213,7 +213,7 @@ def load_transformer(file_path): # triggered by opts.sd_unet change _transformer, _text_encoder_2 = load_flux_bnb(file_path, diffusers_load_config) if _transformer is not None: transformer = _transformer - elif 'nf4' in quant: # TODO fix flux loader for civitai nf4 models + elif 'nf4' in quant: # TODO flux: fix loader for civitai nf4 models from modules.model_flux_nf4 import load_flux_nf4 _transformer, _text_encoder_2 = load_flux_nf4(file_path) if _transformer is not None: @@ -342,7 +342,7 @@ def load_flux(checkpoint_info, diffusers_load_config): # triggered by opts.sd_ch shared.log.debug(f'Load model: type=FLUX cls={cls.__name__} preloaded={list(kwargs)} revision={diffusers_load_config.get("revision", None)}') for c in kwargs: if getattr(kwargs[c], 'quantization_method', None) is not None or getattr(kwargs[c], 'gguf', None) is not None: - shared.log.debug(f'Load model: type=FLUX component={c} dtype={kwargs[c].dtype} quant={getattr(kwargs[c], 'quantization_method', None) or getattr(kwargs[c], 'gguf', None)}') + shared.log.debug(f'Load model: type=FLUX component={c} dtype={kwargs[c].dtype} quant={getattr(kwargs[c], "quantization_method", None) or getattr(kwargs[c], "gguf", None)}') if kwargs[c].dtype == torch.float32 and devices.dtype != torch.float32: try: kwargs[c] = kwargs[c].to(dtype=devices.dtype) diff --git a/modules/model_sd3.py b/modules/model_sd3.py index 36dfbd9b4..1834c573e 100644 --- a/modules/model_sd3.py +++ b/modules/model_sd3.py @@ -1,7 +1,7 @@ import os import diffusers import transformers -from modules import shared, devices, sd_models, sd_unet, model_te, model_quant, model_tools +from modules import shared, devices, sd_models, sd_unet, model_quant, model_tools def load_overrides(kwargs, cache_dir): diff --git a/modules/omnigen/utils.py b/modules/omnigen/utils.py index 5483d6eab..0304e1732 100644 --- a/modules/omnigen/utils.py +++ b/modules/omnigen/utils.py @@ -25,7 +25,6 @@ def update_ema(ema_model, model, decay=0.9999): """ ema_params = dict(ema_model.named_parameters()) for name, param in model.named_parameters(): - # TODO: Consider applying only to params that require_grad to avoid small numerical changes of pos_embed ema_params[name].mul_(decay).add_(param.data, alpha=1 - decay) diff --git a/modules/onnx_impl/pipelines/__init__.py b/modules/onnx_impl/pipelines/__init__.py index ca1ddd2f7..a11b07fc7 100644 --- a/modules/onnx_impl/pipelines/__init__.py +++ b/modules/onnx_impl/pipelines/__init__.py @@ -241,7 +241,7 @@ def run_olive(self, submodels: List[str], in_dir: os.PathLike, out_dir: os.PathL for i in range(len(flow)): flow[i] = flow[i].replace("AutoExecutionProvider", shared.opts.onnx_execution_provider) olive_config["input_model"]["config"]["model_path"] = os.path.abspath(os.path.join(in_dir, submodel, "model.onnx")) - olive_config["systems"]["local_system"]["config"]["accelerators"][0]["device"] = "cpu" if shared.opts.onnx_execution_provider == ExecutionProvider.CPU else "gpu" # TODO: npu + olive_config["systems"]["local_system"]["config"]["accelerators"][0]["device"] = "cpu" if shared.opts.onnx_execution_provider == ExecutionProvider.CPU else "gpu" olive_config["systems"]["local_system"]["config"]["accelerators"][0]["execution_providers"] = [shared.opts.onnx_execution_provider] for pass_key in olive_config["passes"]: diff --git a/modules/pag/pipe_sd.py b/modules/pag/pipe_sd.py index 16f9b5319..11f4fb0cf 100644 --- a/modules/pag/pipe_sd.py +++ b/modules/pag/pipe_sd.py @@ -104,7 +104,6 @@ def __call__( value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) # the output of sdp = (batch, num_heads, seq_len, head_dim) - # TODO: add support for attn.scale when we move to Torch 2.1 hidden_states_org = F.scaled_dot_product_attention( query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False ) @@ -219,7 +218,6 @@ def __call__( value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) # the output of sdp = (batch, num_heads, seq_len, head_dim) - # TODO: add support for attn.scale when we move to Torch 2.1 hidden_states_org = F.scaled_dot_product_attention( query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False ) diff --git a/modules/pag/pipe_sdxl.py b/modules/pag/pipe_sdxl.py index 82ae06c07..3a47af3e5 100644 --- a/modules/pag/pipe_sdxl.py +++ b/modules/pag/pipe_sdxl.py @@ -124,7 +124,6 @@ def __call__( value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) # the output of sdp = (batch, num_heads, seq_len, head_dim) - # TODO: add support for attn.scale when we move to Torch 2.1 hidden_states_org = F.scaled_dot_product_attention( query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False ) @@ -239,7 +238,6 @@ def __call__( value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) # the output of sdp = (batch, num_heads, seq_len, head_dim) - # TODO: add support for attn.scale when we move to Torch 2.1 hidden_states_org = F.scaled_dot_product_attention( query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False ) diff --git a/modules/processing_class.py b/modules/processing_class.py index f502e0dcc..97e5f7b6f 100644 --- a/modules/processing_class.py +++ b/modules/processing_class.py @@ -170,7 +170,7 @@ def __init__(self, self.image_cfg_scale = image_cfg_scale self.scale_by = scale_by self.mask = mask - self.image_mask = mask # TODO remove duplicate mask params + self.image_mask = mask # TODO processing: remove duplicate mask params self.latent_mask = latent_mask self.mask_blur = mask_blur self.inpainting_fill = inpainting_fill diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index d43660ca8..aa820aa34 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -330,13 +330,6 @@ def process_refine(p: processing.StableDiffusionProcessing, output): errors.display(e, 'Processing') modelstats.analyze() - """ # TODO decode using refiner - if not shared.state.interrupted and not shared.state.skipped: - refiner_images = processing_vae.vae_decode(latents=refiner_output.images, model=shared.sd_refiner, full_quality=True, width=max(p.width, p.hr_upscale_to_x), height=max(p.height, p.hr_upscale_to_y)) - for refiner_image in refiner_images: - results.append(refiner_image) - """ - if shared.opts.diffusers_offload_mode == "balanced": shared.sd_refiner = sd_models.apply_balanced_offload(shared.sd_refiner) elif shared.opts.diffusers_move_refiner: diff --git a/modules/pulid/eva_clip/hf_model.py b/modules/pulid/eva_clip/hf_model.py index c4b9fd85b..d148bbff2 100644 --- a/modules/pulid/eva_clip/hf_model.py +++ b/modules/pulid/eva_clip/hf_model.py @@ -31,7 +31,6 @@ class PretrainedConfig: def _camel2snake(s): return re.sub(r'(? self.config.num_train_timesteps: diff --git a/modules/schedulers/scheduler_ufogen.py b/modules/schedulers/scheduler_ufogen.py index e03dec3c0..f4d8aee97 100644 --- a/modules/schedulers/scheduler_ufogen.py +++ b/modules/schedulers/scheduler_ufogen.py @@ -310,14 +310,11 @@ def set_timesteps( self.num_inference_steps = num_inference_steps self.custom_timesteps = False - # TODO: For now, handle special case when num_inference_steps == 1 separately if num_inference_steps == 1: # Set the timestep schedule to num_train_timesteps - 1 rather than 0 # (that is, the one-step timestep schedule is always trailing rather than leading or linspace) timesteps = np.array([self.config.num_train_timesteps - 1], dtype=np.int64) else: - # TODO: For now, retain the DDPM timestep spacing logic - # "linspace", "leading", "trailing" corresponds to annotation of Table 2. of https://arxiv.org/abs/2305.08891 if self.config.timestep_spacing == "linspace": timesteps = ( np.linspace(0, self.config.num_train_timesteps - 1, num_inference_steps) @@ -446,8 +443,6 @@ def step( # Noise is not used on the final timestep of the timestep schedule. # This also means that noise is not used for one-step sampling. if t != self.timesteps[-1]: - # TODO: is this correct? - # Sample prev sample x_{t - 1} ~ q(x_{t - 1} | x_0 = G(x_t, t)) device = model_output.device noise = randn_tensor(model_output.shape, generator=generator, device=device, dtype=model_output.dtype) sqrt_alpha_prod_t_prev = alpha_prod_t_prev**0.5 diff --git a/modules/schedulers/scheduler_vdm.py b/modules/schedulers/scheduler_vdm.py index 543b29ff3..492c30a0c 100644 --- a/modules/schedulers/scheduler_vdm.py +++ b/modules/schedulers/scheduler_vdm.py @@ -147,7 +147,7 @@ def __init__( self.timesteps = torch.from_numpy(self.get_timesteps(len(self))) if num_train_timesteps: alphas_cumprod = self.alphas_cumprod(torch.flip(self.timesteps, dims=(0,))) - alphas = alphas_cumprod[1:] / alphas_cumprod[:-1] # TODO: Might not be exact + alphas = alphas_cumprod[1:] / alphas_cumprod[:-1] self.alphas = torch.cat([alphas_cumprod[:1], alphas]) self.betas = 1 - self.alphas diff --git a/modules/sd_hijack_hypertile.py b/modules/sd_hijack_hypertile.py index dbf977b8d..69c4163dc 100644 --- a/modules/sd_hijack_hypertile.py +++ b/modules/sd_hijack_hypertile.py @@ -112,7 +112,7 @@ def wrapper(*args, **kwargs): out = forward(x, *args[1:], **kwargs) return out if x.ndim == 4: # VAE - # TODO hypertile vae breaks for diffusers when using non-standard sizes + # TODO hypertile: vae breaks when using non-standard sizes if nh * nw > 1: x = rearrange(x, "b c (nh h) (nw w) -> (b nh nw) c h w", nh=nh, nw=nw) out = forward(x, *args[1:], **kwargs) diff --git a/modules/sd_models.py b/modules/sd_models.py index 9daccf11f..087e8ecfc 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -1496,7 +1496,7 @@ def reload_model_weights(sd_model=None, info=None, reuse_dict=False, op='model', unload_model_weights(op=op) sd_model = None timer = Timer() - # TODO implement model in-memory caching + # TODO model loader: implement model in-memory caching state_dict = get_checkpoint_state_dict(checkpoint_info, timer) if not shared.native else None checkpoint_config = sd_models_config.find_checkpoint_config(state_dict, checkpoint_info) timer.record("config") diff --git a/modules/sd_samplers_common.py b/modules/sd_samplers_common.py index 2ac7949a6..a90ceec27 100644 --- a/modules/sd_samplers_common.py +++ b/modules/sd_samplers_common.py @@ -51,16 +51,6 @@ def single_sample_to_image(sample, approximation=None): return Image.new(mode="RGB", size=(512, 512)) if len(sample.shape) == 4 and sample.shape[0]: # likely animatediff latent sample = sample.permute(1, 0, 2, 3)[0] - """ - # TODO remove - if shared.native: # [-x,x] to [-5,5] - sample_max = torch.max(sample) - if sample_max > 5: - sample = sample * (5 / sample_max) - sample_min = torch.min(sample) - if sample_min < -5: - sample = sample * (5 / abs(sample_min)) - """ if approximation == 2: # TAESD if shared.opts.live_preview_downscale and (sample.shape[-1] > 128 or sample.shape[-2] > 128): scale = 128 / max(sample.shape[-1], sample.shape[-2]) diff --git a/modules/sd_unet.py b/modules/sd_unet.py index f730bdb74..deb0b24b0 100644 --- a/modules/sd_unet.py +++ b/modules/sd_unet.py @@ -36,7 +36,7 @@ def load_unet(model): model.prior_pipe.text_encoder = prior_text_encoder.to(devices.device, dtype=devices.dtype) elif "Flux" in model.__class__.__name__ or "StableDiffusion3" in model.__class__.__name__: loaded_unet = shared.opts.sd_unet - sd_models.load_diffuser() # TODO forcing reloading entire model as loading transformers only leads to massive memory usage + sd_models.load_diffuser() # TODO model load: force-reloading entire model as loading transformers only leads to massive memory usage """ from modules.model_flux import load_transformer transformer = load_transformer(unet_dict[shared.opts.sd_unet]) diff --git a/modules/segmoe/segmoe_model.py b/modules/segmoe/segmoe_model.py index 4b96527be..a542c1fbb 100644 --- a/modules/segmoe/segmoe_model.py +++ b/modules/segmoe/segmoe_model.py @@ -136,7 +136,7 @@ def __init__(self, config_or_path, **kwargs) -> Any: memory_format=torch.channels_last, ) - def to(self, *args, **kwargs): # TODO added no-op to avoid error + def to(self, *args, **kwargs): self.pipe.to(*args, **kwargs) def load_from_scratch(self, config: str, **kwargs) -> None: @@ -202,7 +202,6 @@ def load_from_scratch(self, config: str, **kwargs) -> None: self.config["down_idx_start"] = self.down_idx_start self.config["down_idx_end"] = self.down_idx_end - # TODO: Add Support for Scheduler Selection self.pipe.scheduler = DDPMScheduler.from_config(self.pipe.scheduler.config) # Load Experts @@ -242,7 +241,6 @@ def load_from_scratch(self, config: str, **kwargs) -> None: **kwargs, ) - # TODO: Add Support for Scheduler Selection expert.scheduler = DDPMScheduler.from_config( expert.scheduler.config ) diff --git a/modules/xadapter/adapter.py b/modules/xadapter/adapter.py index 69030fae3..4096f71e7 100644 --- a/modules/xadapter/adapter.py +++ b/modules/xadapter/adapter.py @@ -266,8 +266,6 @@ def forward(self, x, t=None): b, c, _, _ = x[-1].shape if t is not None: if not torch.is_tensor(t): - # TODO: this requires sync between CPU and GPU. So try to pass timesteps as tensors if you can - # This would be a good case for the `match` statement (Python 3.10+) is_mps = x[0].device.type == "mps" if isinstance(timestep, float): dtype = torch.float32 if is_mps else torch.float64 diff --git a/modules/xadapter/unet_adapter.py b/modules/xadapter/unet_adapter.py index 5022f1847..5890c7749 100644 --- a/modules/xadapter/unet_adapter.py +++ b/modules/xadapter/unet_adapter.py @@ -807,8 +807,6 @@ def forward( # 1. time timesteps = timestep if not torch.is_tensor(timesteps): - # TODO: this requires sync between CPU and GPU. So try to pass timesteps as tensors if you can - # This would be a good case for the `match` statement (Python 3.10+) is_mps = sample.device.type == "mps" if isinstance(timestep, float): dtype = torch.float32 if is_mps else torch.float64 @@ -1012,7 +1010,7 @@ def forward( if is_bridge: if up_block_additional_residual[0].shape != sample.shape: - pass # TODO VM patch + pass elif fusion_guidance_scale is not None: sample = sample + fusion_guidance_scale * (up_block_additional_residual.pop(0) - sample) else: @@ -1051,7 +1049,7 @@ def forward( ################# bridge usage ################# if is_bridge and len(up_block_additional_residual) > 0: if sample.shape != up_block_additional_residual[0].shape: - pass # TODO VM PATCH + pass elif fusion_guidance_scale is not None: sample = sample + fusion_guidance_scale * (up_block_additional_residual.pop(0) - sample) else: diff --git a/scripts/instantir.py b/scripts/instantir.py index 5eb7d503a..6ab7733fe 100644 --- a/scripts/instantir.py +++ b/scripts/instantir.py @@ -80,7 +80,7 @@ def run(self, p: processing.StableDiffusionProcessing, *args): # pylint: disable devices.torch_gc() def after(self, p: processing.StableDiffusionProcessing, processed: processing.Processed, *args): # pylint: disable=arguments-differ, unused-argument - # TODO instantir is a mess to unload + # TODO instantir: a mess to unload """ if self.orig_pipe is None: return processed diff --git a/scripts/stablevideodiffusion.py b/scripts/stablevideodiffusion.py index 127466701..c1283e1b6 100644 --- a/scripts/stablevideodiffusion.py +++ b/scripts/stablevideodiffusion.py @@ -75,7 +75,7 @@ def run(self, p: processing.StableDiffusionProcessing, model, num_frames, overri if model_name != model_loaded or c != 'StableVideoDiffusionPipeline': shared.opts.sd_model_checkpoint = model_path sd_models.reload_model_weights() - shared.sd_model = shared.sd_model.to(torch.float32) # TODO svd runs in fp32 + shared.sd_model = shared.sd_model.to(torch.float32) # TODO svd: runs in fp32 causing dtype mismatch # set params if override_resolution: From e699e45086a7e10530396839d22f767046401ef4 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sun, 22 Dec 2024 22:58:48 +0300 Subject: [PATCH 134/249] Update OpenVINO to 2024.6.0 --- CHANGELOG.md | 2 +- installer.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 83eda417d..d88850eb7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -176,7 +176,7 @@ All-in-all, we're around ~160 commits worth of updates, check changelog for full - `GGUF` with pre-quantized weights - Switch `GGUF` loader from custom to diffuser native - **IPEX**: update to IPEX 2.5.10+xpu -- **OpenVINO**: update to 2024.5.0 +- **OpenVINO**: update to 2024.6.0 - **Sampler** improvements - UniPC, DEIS, SA, DPM-Multistep: allow FlowMatch sigma method and prediction type - Euler FlowMatch: add sigma methods (*karras/exponential/betas*) diff --git a/installer.py b/installer.py index 55f23c27f..4e99bd054 100644 --- a/installer.py +++ b/installer.py @@ -652,12 +652,12 @@ def install_ipex(torch_command): def install_openvino(torch_command): - check_python(supported_minors=[8, 9, 10, 11, 12], reason='OpenVINO backend requires Python 3.9, 3.10 or 3.11') + check_python(supported_minors=[9, 10, 11, 12], reason='OpenVINO backend requires Python 3.9, 3.10 or 3.11') log.info('OpenVINO: selected') torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.3.1+cpu torchvision==0.18.1+cpu --index-url https://download.pytorch.org/whl/cpu') - install(os.environ.get('OPENVINO_PACKAGE', 'openvino==2024.5.0'), 'openvino') + install(os.environ.get('OPENVINO_PACKAGE', 'openvino==2024.6.0'), 'openvino') install(os.environ.get('ONNXRUNTIME_PACKAGE', 'onnxruntime-openvino'), 'onnxruntime-openvino', ignore=True) - install('nncf==2.12.0', 'nncf') + install('nncf==2.14.1', 'nncf') os.environ.setdefault('PYTORCH_TRACING_MODE', 'TORCHFX') if os.environ.get("NEOReadDebugKeys", None) is None: os.environ.setdefault('NEOReadDebugKeys', '1') From a53009e1ee8767840e4968a473f69c423540031a Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sun, 22 Dec 2024 23:45:19 +0300 Subject: [PATCH 135/249] Don't install OpenVINO without --use-openvino --- installer.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/installer.py b/installer.py index 4e99bd054..111e92a19 100644 --- a/installer.py +++ b/installer.py @@ -734,8 +734,6 @@ def check_torch(): torch_command = install_rocm_zluda() elif is_ipex_available: torch_command = install_ipex(torch_command) - elif allow_openvino: - torch_command = install_openvino(torch_command) else: machine = platform.machine() From 0c7d7941136e1617e55966f4a268dcb1053da652 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Mon, 23 Dec 2024 00:16:34 +0300 Subject: [PATCH 136/249] OpenVINO disable model caching by default --- CHANGELOG.md | 4 +++- installer.py | 5 ++++- modules/shared.py | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d88850eb7..88a738a86 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -176,7 +176,9 @@ All-in-all, we're around ~160 commits worth of updates, check changelog for full - `GGUF` with pre-quantized weights - Switch `GGUF` loader from custom to diffuser native - **IPEX**: update to IPEX 2.5.10+xpu -- **OpenVINO**: update to 2024.6.0 +- **OpenVINO**: + - update to 2024.6.0 + - disable model caching by default - **Sampler** improvements - UniPC, DEIS, SA, DPM-Multistep: allow FlowMatch sigma method and prediction type - Euler FlowMatch: add sigma methods (*karras/exponential/betas*) diff --git a/installer.py b/installer.py index 111e92a19..26d9a3a11 100644 --- a/installer.py +++ b/installer.py @@ -654,7 +654,10 @@ def install_ipex(torch_command): def install_openvino(torch_command): check_python(supported_minors=[9, 10, 11, 12], reason='OpenVINO backend requires Python 3.9, 3.10 or 3.11') log.info('OpenVINO: selected') - torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.3.1+cpu torchvision==0.18.1+cpu --index-url https://download.pytorch.org/whl/cpu') + if sys.platform == 'darwin': + torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.3.1 torchvision==0.18.1') + else: + torch_command = os.environ.get('TORCH_COMMAND', 'torch==2.3.1+cpu torchvision==0.18.1+cpu --index-url https://download.pytorch.org/whl/cpu') install(os.environ.get('OPENVINO_PACKAGE', 'openvino==2024.6.0'), 'openvino') install(os.environ.get('ONNXRUNTIME_PACKAGE', 'onnxruntime-openvino'), 'onnxruntime-openvino', ignore=True) install('nncf==2.14.1', 'nncf') diff --git a/modules/shared.py b/modules/shared.py index 78ff9dc3e..ee38b236a 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -573,7 +573,7 @@ def get_default_modes(): "openvino_sep": OptionInfo("

OpenVINO

", "", gr.HTML, {"visible": cmd_opts.use_openvino}), "openvino_devices": OptionInfo([], "OpenVINO devices to use", gr.CheckboxGroup, {"choices": get_openvino_device_list() if cmd_opts.use_openvino else [], "visible": cmd_opts.use_openvino}), # pylint: disable=E0606 "openvino_accuracy": OptionInfo("performance", "OpenVINO accuracy mode", gr.Radio, {"choices": ['performance', 'accuracy'], "visible": cmd_opts.use_openvino}), - "openvino_disable_model_caching": OptionInfo(False, "OpenVINO disable model caching", gr.Checkbox, {"visible": cmd_opts.use_openvino}), + "openvino_disable_model_caching": OptionInfo(True, "OpenVINO disable model caching", gr.Checkbox, {"visible": cmd_opts.use_openvino}), "openvino_disable_memory_cleanup": OptionInfo(True, "OpenVINO disable memory cleanup after compile", gr.Checkbox, {"visible": cmd_opts.use_openvino}), "directml_sep": OptionInfo("

DirectML

", "", gr.HTML, {"visible": devices.backend == "directml"}), From dcb91499d37a33c5aa0ed823d00a7cb73cdc199f Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 22 Dec 2024 17:38:51 -0500 Subject: [PATCH 137/249] fix pag with batch count Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 2 ++ modules/pag/__init__.py | 21 +++++++++++---------- scripts/animatediff.py | 2 +- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 88a738a86..703804ab3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -220,6 +220,8 @@ All-in-all, we're around ~160 commits worth of updates, check changelog for full - fix svd image2video - fix gallery display during generate - fix wildcards replacement to be unique +- fix animatediff-xl +- fix pag with batch count ## Update for 2024-11-21 diff --git a/modules/pag/__init__.py b/modules/pag/__init__.py index b7a56c40d..a72f7825d 100644 --- a/modules/pag/__init__.py +++ b/modules/pag/__init__.py @@ -12,29 +12,29 @@ def apply(p: processing.StableDiffusionProcessing): # pylint: disable=arguments- global orig_pipeline # pylint: disable=global-statement if not shared.native: return None - c = shared.sd_model.__class__ if shared.sd_loaded else None - if c == StableDiffusionPAGPipeline or c == StableDiffusionXLPAGPipeline: - unapply() + cls = shared.sd_model.__class__ if shared.sd_loaded else None + if cls == StableDiffusionPAGPipeline or cls == StableDiffusionXLPAGPipeline: + cls = unapply() if p.pag_scale == 0: return - if 'PAG' in shared.sd_model.__class__.__name__: + if 'PAG' in cls.__name__: pass - elif detect.is_sd15(c): + elif detect.is_sd15(cls): if sd_models.get_diffusers_task(shared.sd_model) != sd_models.DiffusersTaskType.TEXT_2_IMAGE: - shared.log.warning(f'PAG: pipeline={c} not implemented') + shared.log.warning(f'PAG: pipeline={cls.__name__} not implemented') return None orig_pipeline = shared.sd_model shared.sd_model = sd_models.switch_pipe(StableDiffusionPAGPipeline, shared.sd_model) - elif detect.is_sdxl(c): + elif detect.is_sdxl(cls): if sd_models.get_diffusers_task(shared.sd_model) != sd_models.DiffusersTaskType.TEXT_2_IMAGE: - shared.log.warning(f'PAG: pipeline={c} not implemented') + shared.log.warning(f'PAG: pipeline={cls.__name__} not implemented') return None orig_pipeline = shared.sd_model shared.sd_model = sd_models.switch_pipe(StableDiffusionXLPAGPipeline, shared.sd_model) - elif detect.is_f1(c): + elif detect.is_f1(cls): p.task_args['true_cfg_scale'] = p.pag_scale else: - shared.log.warning(f'PAG: pipeline={c} required={StableDiffusionPipeline.__name__}') + shared.log.warning(f'PAG: pipeline={cls.__name__} required={StableDiffusionPipeline.__name__}') return None p.task_args['pag_scale'] = p.pag_scale @@ -54,3 +54,4 @@ def unapply(): if orig_pipeline is not None: shared.sd_model = orig_pipeline orig_pipeline = None + return shared.sd_model.__class__ diff --git a/scripts/animatediff.py b/scripts/animatediff.py index 6c29f3fa5..f44c85bb7 100644 --- a/scripts/animatediff.py +++ b/scripts/animatediff.py @@ -182,7 +182,7 @@ def set_free_init(method, iters, order, spatial, temporal): def set_free_noise(frames): context_length = 16 context_stride = 4 - if frames >= context_length: + if frames >= context_length and hasattr(shared.sd_model, 'enable_free_noise'): shared.log.debug(f'AnimateDiff free noise: frames={frames} context={context_length} stride={context_stride}') shared.sd_model.enable_free_noise(context_length=context_length, context_stride=context_stride) From acde6f2a20fc86e3e10d98a5f6613f2dd0daf2d4 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 23 Dec 2024 11:56:51 -0500 Subject: [PATCH 138/249] new ltxvideo, offloading, quantization Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 23 +++++----- installer.py | 2 +- modules/processing.py | 17 +++++-- modules/processing_diffusers.py | 3 ++ modules/processing_vae.py | 12 +++-- scripts/ltxvideo.py | 78 ++++++++++++++++++++++++++++----- 6 files changed, 105 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 703804ab3..0dac17552 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,8 @@ # Change Log for SD.Next -## Update for 2024-12-22 +## Update for 2024-12-23 -### Highlights for 2024-12-22 +### Highlights for 2024-12-23 ### SD.Next Xmass edition: *What's new?* @@ -32,7 +32,7 @@ All-in-all, we're around ~160 commits worth of updates, check changelog for full [ReadMe](https://github.com/vladmandic/automatic/blob/master/README.md) | [ChangeLog](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) | [Docs](https://vladmandic.github.io/sdnext-docs/) | [WiKi](https://github.com/vladmandic/automatic/wiki) | [Discord](https://discord.com/invite/sd-next-federal-batch-inspectors-1101998836328697867) -## Details for 2024-12-22 +## Details for 2024-12-23 ### New models and integrations @@ -93,17 +93,20 @@ All-in-all, we're around ~160 commits worth of updates, check changelog for full ### Video models -- [Lightricks LTX-Video](https://huggingface.co/Lightricks/LTX-Video) - model size: 27.75gb +- [Lightricks LTX-Video](https://huggingface.co/Lightricks/LTX-Video) + model size: 27.75gb + support for 0.9.0, 0.9.1 and custom safetensor-based models with full quantization and offloading support support for text-to-video and image-to-video, to use, select in *scripts -> ltx-video* - *refrence values*: steps 50, width 704, height 512, frames 161, guidance scale 3.0 + *refrence values*: steps 50, width 704, height 512, frames 161, guidance scale 3.0 - [Hunyuan Video](https://huggingface.co/tencent/HunyuanVideo) - model size: 40.92gb + model size: 40.92gb support for text-to-video, to use, select in *scripts -> hunyuan video* - *refrence values*: steps 50, width 1280, height 720, frames 129, guidance scale 6.0 -- [Genmo Mochi.1 Preview](https://huggingface.co/genmo/mochi-1-preview) + basic support only + *refrence values*: steps 50, width 1280, height 720, frames 129, guidance scale 6.0 +- [Genmo Mochi.1 Preview](https://huggingface.co/genmo/mochi-1-preview) support for text-to-video, to use, select in *scripts -> mochi.1 video* - *refrence values*: steps 64, width 848, height 480, frames 19, guidance scale 4.5 + basic support only + *refrence values*: steps 64, width 848, height 480, frames 19, guidance scale 4.5 *Notes*: - all video models are very large and resource intensive! diff --git a/installer.py b/installer.py index 26d9a3a11..46ccce92c 100644 --- a/installer.py +++ b/installer.py @@ -459,7 +459,7 @@ def check_python(supported_minors=[9, 10, 11, 12], reason=None): def check_diffusers(): if args.skip_all or args.skip_git: return - sha = '233dffdc3f56b26abaaba8363a5dd30dab7f0e40' # diffusers commit hash + sha = '4b557132ce955d58fd84572c03e79f43bdc91450' # diffusers commit hash pkg = pkg_resources.working_set.by_key.get('diffusers', None) minor = int(pkg.version.split('.')[1] if pkg is not None else 0) cur = opts.get('diffusers_version', '') if minor > 0 else '' diff --git a/modules/processing.py b/modules/processing.py index 6d9b64c17..c85938268 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -357,7 +357,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: for i, sample in enumerate(samples): debug(f'Processing result: index={i+1}/{len(samples)} iteration={n+1}/{p.n_iter}') p.batch_index = i - if type(sample) == Image.Image: + if isinstance(sample, Image.Image) or (isinstance(sample, list) and isinstance(sample[0], Image.Image)): image = sample sample = np.array(sample) else: @@ -399,11 +399,20 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: info = create_infotext(p, p.prompts, p.seeds, p.subseeds, index=i, all_negative_prompts=p.negative_prompts) infotexts.append(info) - image.info["parameters"] = info - output_images.append(image) + if isinstance(image, list): + for img in image: + img.info["parameters"] = info + output_images = image + else: + image.info["parameters"] = info + output_images.append(image) if shared.opts.samples_save and not p.do_not_save_samples and p.outpath_samples is not None: info = create_infotext(p, p.prompts, p.seeds, p.subseeds, index=i) - images.save_image(image, p.outpath_samples, "", p.seeds[i], p.prompts[i], shared.opts.samples_format, info=info, p=p) # main save image + if isinstance(image, list): + for img in image: + images.save_image(img, p.outpath_samples, "", p.seeds[i], p.prompts[i], shared.opts.samples_format, info=info, p=p) # main save image + else: + images.save_image(image, p.outpath_samples, "", p.seeds[i], p.prompts[i], shared.opts.samples_format, info=info, p=p) # main save image if hasattr(p, 'mask_for_overlay') and p.mask_for_overlay and any([shared.opts.save_mask, shared.opts.save_mask_composite, shared.opts.return_mask, shared.opts.return_mask_composite]): image_mask = p.mask_for_overlay.convert('RGB') image1 = image.convert('RGBA').convert('RGBa') diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index aa820aa34..33d875e95 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -361,6 +361,7 @@ def process_decode(p: processing.StableDiffusionProcessing, output): else: width = getattr(p, 'width', 0) height = getattr(p, 'height', 0) + frames = p.task_args.get('num_frames', None) if isinstance(output.images, list): results = [] for i in range(len(output.images)): @@ -370,6 +371,7 @@ def process_decode(p: processing.StableDiffusionProcessing, output): full_quality = p.full_quality, width = width, height = height, + frames = frames, ) for result in list(result_batch): results.append(result) @@ -380,6 +382,7 @@ def process_decode(p: processing.StableDiffusionProcessing, output): full_quality = p.full_quality, width = width, height = height, + frames = frames, ) elif hasattr(output, 'images'): results = output.images diff --git a/modules/processing_vae.py b/modules/processing_vae.py index 5d8f9fc84..1710bd992 100644 --- a/modules/processing_vae.py +++ b/modules/processing_vae.py @@ -203,7 +203,7 @@ def taesd_vae_encode(image): return encoded -def vae_decode(latents, model, output_type='np', full_quality=True, width=None, height=None): +def vae_decode(latents, model, output_type='np', full_quality=True, width=None, height=None, frames=None): t0 = time.time() model = model or shared.sd_model if not hasattr(model, 'vae') and hasattr(model, 'pipe'): @@ -221,7 +221,11 @@ def vae_decode(latents, model, output_type='np', full_quality=True, width=None, shared.log.error('VAE not found in model') return [] - if hasattr(model, "_unpack_latents") and hasattr(model, "vae_scale_factor") and width is not None and height is not None: # FLUX + if hasattr(model, '_unpack_latents') and hasattr(model, 'transformer_spatial_patch_size') and frames is not None: # LTX + latent_num_frames = (frames - 1) // model.vae_temporal_compression_ratio + 1 + latents = model._unpack_latents(latents.unsqueeze(0), latent_num_frames, height // 32, width // 32, model.transformer_spatial_patch_size, model.transformer_temporal_patch_size) # pylint: disable=protected-access + latents = model._denormalize_latents(latents, model.vae.latents_mean, model.vae.latents_std, model.vae.config.scaling_factor) # pylint: disable=protected-access + if hasattr(model, '_unpack_latents') and hasattr(model, "vae_scale_factor") and width is not None and height is not None: # FLUX latents = model._unpack_latents(latents, height, width, model.vae_scale_factor) # pylint: disable=protected-access if len(latents.shape) == 3: # lost a batch dim in hires latents = latents.unsqueeze(0) @@ -238,7 +242,9 @@ def vae_decode(latents, model, output_type='np', full_quality=True, width=None, decoded = taesd_vae_decode(latents=latents) if torch.is_tensor(decoded): - if hasattr(model, 'image_processor'): + if hasattr(model, 'video_processor'): + imgs = model.video_processor.postprocess_video(decoded, output_type='pil') + elif hasattr(model, 'image_processor'): imgs = model.image_processor.postprocess(decoded, output_type=output_type) elif hasattr(model, "vqgan"): imgs = decoded.permute(0, 2, 3, 1).cpu().float().numpy() diff --git a/scripts/ltxvideo.py b/scripts/ltxvideo.py index 148fd4481..007c4f4cc 100644 --- a/scripts/ltxvideo.py +++ b/scripts/ltxvideo.py @@ -1,11 +1,40 @@ +import os import time import torch import gradio as gr import diffusers +import transformers from modules import scripts, processing, shared, images, devices, sd_models, sd_checkpoint, model_quant -repo_id = 'a-r-r-o-w/LTX-Video-diffusers' +repos = { + '0.9.0': 'a-r-r-o-w/LTX-Video-diffusers', + '0.9.1': 'a-r-r-o-w/LTX-Video-0.9.1-diffusers', + 'custom': None, +} + + +def load_quants(kwargs, repo_id): + if len(shared.opts.bnb_quantization) > 0: + quant_args = {} + quant_args = model_quant.create_bnb_config(quant_args) + quant_args = model_quant.create_ao_config(quant_args) + if not quant_args: + return kwargs + model_quant.load_bnb(f'Load model: type=LTX quant={quant_args}') + if 'Model' in shared.opts.bnb_quantization and 'transformer' not in kwargs: + kwargs['transformer'] = diffusers.LTXVideoTransformer3DModel.from_pretrained(repo_id, subfolder="transformer", cache_dir=shared.opts.hfcache_dir, torch_dtype=devices.dtype, **quant_args) + shared.log.debug(f'Quantization: module=transformer type=bnb dtype={shared.opts.bnb_quantization_type} storage={shared.opts.bnb_quantization_storage}') + if 'Text Encoder' in shared.opts.bnb_quantization and 'text_encoder_3' not in kwargs: + kwargs['text_encoder'] = transformers.T5EncoderModel.from_pretrained(repo_id, subfolder="text_encoder", cache_dir=shared.opts.hfcache_dir, torch_dtype=devices.dtype, **quant_args) + shared.log.debug(f'Quantization: module=t5 type=bnb dtype={shared.opts.bnb_quantization_type} storage={shared.opts.bnb_quantization_storage}') + return kwargs + + +def hijack_decode(*args, **kwargs): + shared.log.debug('Video: decode') + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) + return shared.sd_model.vae.orig_decode(*args, **kwargs) class Script(scripts.Script): @@ -24,11 +53,19 @@ def video_type_change(video_type): gr.update(visible=video_type == 'MP4'), gr.update(visible=video_type == 'MP4'), ] + def model_change(model): + return gr.update(visible=model == 'custom') with gr.Row(): gr.HTML('  LTX Video
') + with gr.Row(): + model = gr.Dropdown(label='LTX Model', choices=list(repos), value='0.9.1') + decode = gr.Dropdown(label='Decode', choices=['diffusers', 'native'], value='diffusers', visible=False) with gr.Row(): num_frames = gr.Slider(label='Frames', minimum=9, maximum=257, step=1, value=41) + sampler = gr.Checkbox(label='Override sampler', value=True) + with gr.Row(): + model_custom = gr.Textbox(value='', label='Path to model file', visible=False) with gr.Row(): video_type = gr.Dropdown(label='Video file', choices=['None', 'GIF', 'PNG', 'MP4'], value='None') duration = gr.Slider(label='Duration', minimum=0.25, maximum=10, step=0.25, value=2, visible=False) @@ -37,9 +74,10 @@ def video_type_change(video_type): mp4_pad = gr.Slider(label='Pad frames', minimum=0, maximum=24, step=1, value=1, visible=False) mp4_interpolate = gr.Slider(label='Interpolate frames', minimum=0, maximum=24, step=1, value=0, visible=False) video_type.change(fn=video_type_change, inputs=[video_type], outputs=[duration, gif_loop, mp4_pad, mp4_interpolate]) - return [num_frames, video_type, duration, gif_loop, mp4_pad, mp4_interpolate] + model.change(fn=model_change, inputs=[model], outputs=[model_custom]) + return [model, model_custom, decode, sampler, num_frames, video_type, duration, gif_loop, mp4_pad, mp4_interpolate] - def run(self, p: processing.StableDiffusionProcessing, num_frames, video_type, duration, gif_loop, mp4_pad, mp4_interpolate): # pylint: disable=arguments-differ, unused-argument + def run(self, p: processing.StableDiffusionProcessing, model, model_custom, decode, sampler, num_frames, video_type, duration, gif_loop, mp4_pad, mp4_interpolate): # pylint: disable=arguments-differ, unused-argument # set params image = getattr(p, 'init_images', None) image = None if image is None or len(image) == 0 else image[0] @@ -49,32 +87,48 @@ def run(self, p: processing.StableDiffusionProcessing, num_frames, video_type, d num_frames = 8 * int(num_frames // 8) + 1 p.width = 32 * int(p.width // 32) p.height = 32 * int(p.height // 32) + processing.fix_seed(p) if image: image = images.resize_image(resize_mode=2, im=image, width=p.width, height=p.height, upscaler_name=None, output_type='pil') p.task_args['image'] = image - p.task_args['output_type'] = 'pil' - p.task_args['generator'] = torch.manual_seed(p.seed) + p.task_args['output_type'] = 'latent' if decode == 'native' else 'pil' + p.task_args['generator'] = torch.Generator(devices.device).manual_seed(p.seed) p.task_args['num_frames'] = num_frames - p.sampler_name = 'Default' p.do_not_save_grid = True + if sampler: + p.sampler_name = 'Default' p.ops.append('video') # load model cls = diffusers.LTXPipeline if image is None else diffusers.LTXImageToVideoPipeline diffusers.LTXTransformer3DModel = diffusers.LTXVideoTransformer3DModel diffusers.AutoencoderKLLTX = diffusers.AutoencoderKLLTXVideo + repo_id = repos[model] + if repo_id is None: + repo_id = model_custom if shared.sd_model.__class__ != cls: sd_models.unload_model_weights() kwargs = {} kwargs = model_quant.create_bnb_config(kwargs) kwargs = model_quant.create_ao_config(kwargs) - shared.sd_model = cls.from_pretrained( - repo_id, - cache_dir = shared.opts.hfcache_dir, - torch_dtype=devices.dtype, - **kwargs - ) + if os.path.isfile(repo_id): + shared.sd_model = cls.from_single_file( + repo_id, + cache_dir = shared.opts.hfcache_dir, + torch_dtype=devices.dtype, + **kwargs + ) + else: + kwargs = load_quants(kwargs, repo_id) + shared.sd_model = cls.from_pretrained( + repo_id, + cache_dir = shared.opts.hfcache_dir, + torch_dtype=devices.dtype, + **kwargs + ) sd_models.set_diffuser_options(shared.sd_model) + shared.sd_model.vae.orig_decode = shared.sd_model.vae.decode + shared.sd_model.vae.decode = hijack_decode shared.sd_model.sd_checkpoint_info = sd_checkpoint.CheckpointInfo(repo_id) shared.sd_model.sd_model_hash = None shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) From fb804f4d2591468239ce71393c1d4fbe29d5c1b3 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 23 Dec 2024 12:23:18 -0500 Subject: [PATCH 139/249] move postprocessing scripts to accordions Signed-off-by: Vladimir Mandic --- TODO.md | 27 +++++----- modules/scripts.py | 58 ++++++++++++---------- modules/shared.py | 2 +- scripts/postprocessing_codeformer.py | 7 +-- scripts/postprocessing_gfpgan.py | 5 +- scripts/postprocessing_upscale.py | 74 ++++++++++++++-------------- scripts/postprocessing_video.py | 67 ++++++++++++------------- 7 files changed, 123 insertions(+), 117 deletions(-) diff --git a/TODO.md b/TODO.md index bf0cecbb4..6d89c838f 100644 --- a/TODO.md +++ b/TODO.md @@ -11,24 +11,21 @@ Main ToDo list can be found at [GitHub projects](https://github.com/users/vladma ## Future Candidates - Flux NF4 loader: - -## Other - - IPAdapter negative: - Control API enhance scripts compatibility - PixelSmith: ## Code TODO -- TODO install: python 3.12.4 or higher cause a mess with pydantic (fixme) -- TODO install: enable ROCm for windows when available (fixme) -- TODO resize image: enable full VAE mode for resize-latent (fixme) -- TODO processing: remove duplicate mask params (fixme) -- TODO flux: fix loader for civitai nf4 models (fixme) -- TODO model loader: implement model in-memory caching (fixme) -- TODO hypertile: vae breaks when using non-standard sizes (fixme) -- TODO model load: force-reloading entire model as loading transformers only leads to massive memory usage (fixme) -- TODO lora load: direct with bnb (fixme) -- TODO: lora make: support quantized flux (fixme) -- TODO control: support scripts via api (fixme) -- TODO modernui: monkey-patch for missing tabs.select event (fixme) +- TODO install: python 3.12.4 or higher cause a mess with pydantic +- TODO install: enable ROCm for windows when available +- TODO resize image: enable full VAE mode for resize-latent +- TODO processing: remove duplicate mask params +- TODO flux: fix loader for civitai nf4 models +- TODO model loader: implement model in-memory caching +- TODO hypertile: vae breaks when using non-standard sizes +- TODO model load: force-reloading entire model as loading transformers only leads to massive memory usage +- TODO lora load: direct with bnb +- TODO lora make: support quantized flux +- TODO control: support scripts via api +- TODO modernui: monkey-patch for missing tabs.select event diff --git a/modules/scripts.py b/modules/scripts.py index 029d0db79..9c410a3b6 100644 --- a/modules/scripts.py +++ b/modules/scripts.py @@ -330,6 +330,7 @@ def __init__(self): self.scripts = [] self.selectable_scripts = [] self.alwayson_scripts = [] + self.auto_processing_scripts = [] self.titles = [] self.infotext_fields = [] self.paste_field_names = [] @@ -337,6 +338,31 @@ def __init__(self): self.is_img2img = False self.inputs = [None] + def add_script(self, script_class, path, is_img2img, is_control): + try: + script = script_class() + script.filename = path + script.is_txt2img = not is_img2img + script.is_img2img = is_img2img + if is_control: # this is messy but show is a legacy function that is not aware of control tab + v1 = script.show(script.is_txt2img) + v2 = script.show(script.is_img2img) + if v1 == AlwaysVisible or v2 == AlwaysVisible: + visibility = AlwaysVisible + else: + visibility = v1 or v2 + else: + visibility = script.show(script.is_img2img) + if visibility == AlwaysVisible: + self.scripts.append(script) + self.alwayson_scripts.append(script) + script.alwayson = True + elif visibility: + self.scripts.append(script) + self.selectable_scripts.append(script) + except Exception as e: + errors.log.error(f'Script initialize: {path} {e}') + def initialize_scripts(self, is_img2img=False, is_control=False): from modules import scripts_auto_postprocessing @@ -351,34 +377,14 @@ def initialize_scripts(self, is_img2img=False, is_control=False): self.scripts.clear() self.alwayson_scripts.clear() self.selectable_scripts.clear() - auto_processing_scripts = scripts_auto_postprocessing.create_auto_preprocessing_script_data() + self.auto_processing_scripts = scripts_auto_postprocessing.create_auto_preprocessing_script_data() - all_scripts = auto_processing_scripts + scripts_data - sorted_scripts = sorted(all_scripts, key=lambda x: x.script_class().title().lower()) + sorted_scripts = sorted(scripts_data, key=lambda x: x.script_class().title().lower()) for script_class, path, _basedir, _script_module in sorted_scripts: - try: - script = script_class() - script.filename = path - script.is_txt2img = not is_img2img - script.is_img2img = is_img2img - if is_control: # this is messy but show is a legacy function that is not aware of control tab - v1 = script.show(script.is_txt2img) - v2 = script.show(script.is_img2img) - if v1 == AlwaysVisible or v2 == AlwaysVisible: - visibility = AlwaysVisible - else: - visibility = v1 or v2 - else: - visibility = script.show(script.is_img2img) - if visibility == AlwaysVisible: - self.scripts.append(script) - self.alwayson_scripts.append(script) - script.alwayson = True - elif visibility: - self.scripts.append(script) - self.selectable_scripts.append(script) - except Exception as e: - errors.log.error(f'Script initialize: {path} {e}') + self.add_script(script_class, path, is_img2img, is_control) + sorted_scripts = sorted(self.auto_processing_scripts, key=lambda x: x.script_class().title().lower()) + for script_class, path, _basedir, _script_module in sorted_scripts: + self.add_script(script_class, path, is_img2img, is_control) def prepare_ui(self): self.inputs = [None] diff --git a/modules/shared.py b/modules/shared.py index ee38b236a..c341deb69 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -837,7 +837,7 @@ def get_default_modes(): options_templates.update(options_section(('postprocessing', "Postprocessing"), { 'postprocessing_enable_in_main_ui': OptionInfo([], "Additional postprocessing operations", gr.Dropdown, lambda: {"multiselect":True, "choices": [x.name for x in shared_items.postprocessing_scripts()]}), - 'postprocessing_operation_order': OptionInfo([], "Postprocessing operation order", gr.Dropdown, lambda: {"multiselect":True, "choices": [x.name for x in shared_items.postprocessing_scripts()]}), + 'postprocessing_operation_order': OptionInfo([], "Postprocessing operation order", gr.Dropdown, lambda: {"multiselect":True, "choices": [x.name for x in shared_items.postprocessing_scripts()], "visible": False }), "postprocessing_sep_img2img": OptionInfo("

Inpaint

", "", gr.HTML), "img2img_color_correction": OptionInfo(False, "Apply color correction"), diff --git a/scripts/postprocessing_codeformer.py b/scripts/postprocessing_codeformer.py index 3e5fd451d..8ed677736 100644 --- a/scripts/postprocessing_codeformer.py +++ b/scripts/postprocessing_codeformer.py @@ -10,9 +10,10 @@ class ScriptPostprocessingCodeFormer(scripts_postprocessing.ScriptPostprocessing order = 3000 def ui(self): - with gr.Row(): - codeformer_visibility = gr.Slider(minimum=0.0, maximum=1.0, step=0.01, label="Strength", value=0.0, elem_id="extras_codeformer_visibility") - codeformer_weight = gr.Slider(minimum=0.0, maximum=1.0, step=0.01, label="Weight", value=0.2, elem_id="extras_codeformer_weight") + with gr.Accordion('Restore faces: CodeFormer', open = False): + with gr.Row(): + codeformer_visibility = gr.Slider(minimum=0.0, maximum=1.0, step=0.01, label="Strength", value=0.0, elem_id="extras_codeformer_visibility") + codeformer_weight = gr.Slider(minimum=0.0, maximum=1.0, step=0.01, label="Weight", value=0.2, elem_id="extras_codeformer_weight") return { "codeformer_visibility": codeformer_visibility, "codeformer_weight": codeformer_weight } def process(self, pp: scripts_postprocessing.PostprocessedImage, codeformer_visibility, codeformer_weight): # pylint: disable=arguments-differ diff --git a/scripts/postprocessing_gfpgan.py b/scripts/postprocessing_gfpgan.py index a69f97c9e..1e17c2b16 100644 --- a/scripts/postprocessing_gfpgan.py +++ b/scripts/postprocessing_gfpgan.py @@ -9,8 +9,9 @@ class ScriptPostprocessingGfpGan(scripts_postprocessing.ScriptPostprocessing): order = 2000 def ui(self): - with gr.Row(): - gfpgan_visibility = gr.Slider(minimum=0.0, maximum=1.0, step=0.001, label="Strength", value=0, elem_id="extras_gfpgan_visibility") + with gr.Accordion('Restore faces: GFPGan', open = False): + with gr.Row(): + gfpgan_visibility = gr.Slider(minimum=0.0, maximum=1.0, step=0.001, label="Strength", value=0, elem_id="extras_gfpgan_visibility") return { "gfpgan_visibility": gfpgan_visibility } def process(self, pp: scripts_postprocessing.PostprocessedImage, gfpgan_visibility): # pylint: disable=arguments-differ diff --git a/scripts/postprocessing_upscale.py b/scripts/postprocessing_upscale.py index 0e07bc847..104a0fb37 100644 --- a/scripts/postprocessing_upscale.py +++ b/scripts/postprocessing_upscale.py @@ -10,43 +10,43 @@ class ScriptPostprocessingUpscale(scripts_postprocessing.ScriptPostprocessing): order = 1000 def ui(self): - selected_tab = gr.State(value=0) # pylint: disable=abstract-class-instantiated - - with gr.Column(): - with gr.Row(elem_id="extras_upscale"): - with gr.Tabs(elem_id="extras_resize_mode"): - with gr.TabItem('Scale by', elem_id="extras_scale_by_tab") as tab_scale_by: - upscaling_resize = gr.Slider(minimum=0.1, maximum=8.0, step=0.05, label="Resize", value=2.0, elem_id="extras_upscaling_resize") - - with gr.TabItem('Scale to', elem_id="extras_scale_to_tab") as tab_scale_to: - with gr.Row(): - with gr.Row(elem_id="upscaling_column_size"): - upscaling_resize_w = gr.Slider(minimum=64, maximum=4096, step=8, label="Width", value=1024, elem_id="extras_upscaling_resize_w") - upscaling_resize_h = gr.Slider(minimum=64, maximum=4096, step=8, label="Height", value=1024, elem_id="extras_upscaling_resize_h") - upscaling_res_switch_btn = ToolButton(value=symbols.switch, elem_id="upscaling_res_switch_btn") - upscaling_crop = gr.Checkbox(label='Crop to fit', value=True, elem_id="extras_upscaling_crop") - - with gr.Row(): - extras_upscaler_1 = gr.Dropdown(label='Upscaler', elem_id="extras_upscaler_1", choices=[x.name for x in shared.sd_upscalers], value=shared.sd_upscalers[0].name) - - with gr.Row(): - extras_upscaler_2 = gr.Dropdown(label='Refine Upscaler', elem_id="extras_upscaler_2", choices=[x.name for x in shared.sd_upscalers], value=shared.sd_upscalers[0].name) - extras_upscaler_2_visibility = gr.Slider(minimum=0.0, maximum=1.0, step=0.001, label="Upscaler 2 visibility", value=0.0, elem_id="extras_upscaler_2_visibility") - - upscaling_res_switch_btn.click(lambda w, h: (h, w), inputs=[upscaling_resize_w, upscaling_resize_h], outputs=[upscaling_resize_w, upscaling_resize_h], show_progress=False) - tab_scale_by.select(fn=lambda: 0, inputs=[], outputs=[selected_tab]) - tab_scale_to.select(fn=lambda: 1, inputs=[], outputs=[selected_tab]) - - return { - "upscale_mode": selected_tab, - "upscale_by": upscaling_resize, - "upscale_to_width": upscaling_resize_w, - "upscale_to_height": upscaling_resize_h, - "upscale_crop": upscaling_crop, - "upscaler_1_name": extras_upscaler_1, - "upscaler_2_name": extras_upscaler_2, - "upscaler_2_visibility": extras_upscaler_2_visibility, - } + with gr.Accordion('Postprocess Upscale', open = False): + selected_tab = gr.State(value=0) # pylint: disable=abstract-class-instantiated + with gr.Column(): + with gr.Row(elem_id="extras_upscale"): + with gr.Tabs(elem_id="extras_resize_mode"): + with gr.TabItem('Scale by', elem_id="extras_scale_by_tab") as tab_scale_by: + upscaling_resize = gr.Slider(minimum=0.1, maximum=8.0, step=0.05, label="Resize", value=2.0, elem_id="extras_upscaling_resize") + + with gr.TabItem('Scale to', elem_id="extras_scale_to_tab") as tab_scale_to: + with gr.Row(): + with gr.Row(elem_id="upscaling_column_size"): + upscaling_resize_w = gr.Slider(minimum=64, maximum=4096, step=8, label="Width", value=1024, elem_id="extras_upscaling_resize_w") + upscaling_resize_h = gr.Slider(minimum=64, maximum=4096, step=8, label="Height", value=1024, elem_id="extras_upscaling_resize_h") + upscaling_res_switch_btn = ToolButton(value=symbols.switch, elem_id="upscaling_res_switch_btn") + upscaling_crop = gr.Checkbox(label='Crop to fit', value=True, elem_id="extras_upscaling_crop") + + with gr.Row(): + extras_upscaler_1 = gr.Dropdown(label='Upscaler', elem_id="extras_upscaler_1", choices=[x.name for x in shared.sd_upscalers], value=shared.sd_upscalers[0].name) + + with gr.Row(): + extras_upscaler_2 = gr.Dropdown(label='Refine Upscaler', elem_id="extras_upscaler_2", choices=[x.name for x in shared.sd_upscalers], value=shared.sd_upscalers[0].name) + extras_upscaler_2_visibility = gr.Slider(minimum=0.0, maximum=1.0, step=0.001, label="Upscaler 2 visibility", value=0.0, elem_id="extras_upscaler_2_visibility") + + upscaling_res_switch_btn.click(lambda w, h: (h, w), inputs=[upscaling_resize_w, upscaling_resize_h], outputs=[upscaling_resize_w, upscaling_resize_h], show_progress=False) + tab_scale_by.select(fn=lambda: 0, inputs=[], outputs=[selected_tab]) + tab_scale_to.select(fn=lambda: 1, inputs=[], outputs=[selected_tab]) + + return { + "upscale_mode": selected_tab, + "upscale_by": upscaling_resize, + "upscale_to_width": upscaling_resize_w, + "upscale_to_height": upscaling_resize_h, + "upscale_crop": upscaling_crop, + "upscaler_1_name": extras_upscaler_1, + "upscaler_2_name": extras_upscaler_2, + "upscaler_2_visibility": extras_upscaler_2_visibility, + } def upscale(self, image, info, upscaler, upscale_mode, upscale_by, upscale_to_width, upscale_to_height, upscale_crop): if upscale_mode == 1: diff --git a/scripts/postprocessing_video.py b/scripts/postprocessing_video.py index 8c266639c..75d375572 100644 --- a/scripts/postprocessing_video.py +++ b/scripts/postprocessing_video.py @@ -7,40 +7,41 @@ class ScriptPostprocessingUpscale(scripts_postprocessing.ScriptPostprocessing): name = "Video" def ui(self): - def video_type_change(video_type): - return [ - gr.update(visible=video_type != 'None'), - gr.update(visible=video_type == 'GIF' or video_type == 'PNG'), - gr.update(visible=video_type == 'MP4'), - gr.update(visible=video_type == 'MP4'), - gr.update(visible=video_type == 'MP4'), - gr.update(visible=video_type == 'MP4'), - ] + with gr.Accordion('Create video', open = False): + def video_type_change(video_type): + return [ + gr.update(visible=video_type != 'None'), + gr.update(visible=video_type == 'GIF' or video_type == 'PNG'), + gr.update(visible=video_type == 'MP4'), + gr.update(visible=video_type == 'MP4'), + gr.update(visible=video_type == 'MP4'), + gr.update(visible=video_type == 'MP4'), + ] - with gr.Row(): - gr.HTML("  Video
") - with gr.Row(): - video_type = gr.Dropdown(label='Video file', choices=['None', 'GIF', 'PNG', 'MP4'], value='None', elem_id="extras_video_type") - duration = gr.Slider(label='Duration', minimum=0.25, maximum=10, step=0.25, value=2, visible=False, elem_id="extras_video_duration") - with gr.Row(): - loop = gr.Checkbox(label='Loop', value=True, visible=False, elem_id="extras_video_loop") - pad = gr.Slider(label='Pad frames', minimum=0, maximum=24, step=1, value=1, visible=False, elem_id="extras_video_pad") - interpolate = gr.Slider(label='Interpolate frames', minimum=0, maximum=24, step=1, value=0, visible=False, elem_id="extras_video_interpolate") - scale = gr.Slider(label='Rescale', minimum=0.5, maximum=2, step=0.05, value=1, visible=False, elem_id="extras_video_scale") - change = gr.Slider(label='Frame change sensitivity', minimum=0, maximum=1, step=0.05, value=0.3, visible=False, elem_id="extras_video_change") - with gr.Row(): - filename = gr.Textbox(label='Filename', placeholder='enter filename', lines=1, elem_id="extras_video_filename") - video_type.change(fn=video_type_change, inputs=[video_type], outputs=[duration, loop, pad, interpolate, scale, change]) - return { - "filename": filename, - "video_type": video_type, - "duration": duration, - "loop": loop, - "pad": pad, - "interpolate": interpolate, - "scale": scale, - "change": change, - } + with gr.Row(): + gr.HTML("  Video
") + with gr.Row(): + video_type = gr.Dropdown(label='Video file', choices=['None', 'GIF', 'PNG', 'MP4'], value='None', elem_id="extras_video_type") + duration = gr.Slider(label='Duration', minimum=0.25, maximum=10, step=0.25, value=2, visible=False, elem_id="extras_video_duration") + with gr.Row(): + loop = gr.Checkbox(label='Loop', value=True, visible=False, elem_id="extras_video_loop") + pad = gr.Slider(label='Pad frames', minimum=0, maximum=24, step=1, value=1, visible=False, elem_id="extras_video_pad") + interpolate = gr.Slider(label='Interpolate frames', minimum=0, maximum=24, step=1, value=0, visible=False, elem_id="extras_video_interpolate") + scale = gr.Slider(label='Rescale', minimum=0.5, maximum=2, step=0.05, value=1, visible=False, elem_id="extras_video_scale") + change = gr.Slider(label='Frame change sensitivity', minimum=0, maximum=1, step=0.05, value=0.3, visible=False, elem_id="extras_video_change") + with gr.Row(): + filename = gr.Textbox(label='Filename', placeholder='enter filename', lines=1, elem_id="extras_video_filename") + video_type.change(fn=video_type_change, inputs=[video_type], outputs=[duration, loop, pad, interpolate, scale, change]) + return { + "filename": filename, + "video_type": video_type, + "duration": duration, + "loop": loop, + "pad": pad, + "interpolate": interpolate, + "scale": scale, + "change": change, + } def postprocess(self, images, filename, video_type, duration, loop, pad, interpolate, scale, change): # pylint: disable=arguments-differ filename = filename.strip() if filename is not None else '' From 90a19a260ee5f06017063b85140e9bcb9e6200ae Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 23 Dec 2024 12:41:02 -0500 Subject: [PATCH 140/249] css tweaks Signed-off-by: Vladimir Mandic --- javascript/base.css | 4 ++-- javascript/sdnext.css | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/javascript/base.css b/javascript/base.css index 6c18ad7c5..a30a71845 100644 --- a/javascript/base.css +++ b/javascript/base.css @@ -79,7 +79,7 @@ table.settings-value-table td { padding: 0.4em; border: 1px solid #ccc; max-widt /* extra networks */ .extra-networks > div { margin: 0; border-bottom: none !important; } -.extra-networks .second-line { display: flex; width: -moz-available; width: -webkit-fill-available; gap: 0.3em; box-shadow: var(--input-shadow); } +.extra-networks .second-line { display: flex; width: -moz-available; width: -webkit-fill-available; gap: 0.3em; box-shadow: var(--input-shadow); margin-bottom: 2px; } .extra-networks .search { flex: 1; } .extra-networks .description { flex: 3; } .extra-networks .tab-nav > button { margin-right: 0; height: 24px; padding: 2px 4px 2px 4px; } @@ -88,7 +88,7 @@ table.settings-value-table td { padding: 0.4em; border: 1px solid #ccc; max-widt .extra-networks .custom-button { width: 120px; width: 100%; background: none; justify-content: left; text-align: left; padding: 2px 8px 2px 16px; text-indent: -8px; box-shadow: none; line-break: auto; } .extra-networks .custom-button:hover { background: var(--button-primary-background-fill) } .extra-networks-tab { padding: 0 !important; } -.extra-network-subdirs { background: var(--input-background-fill); overflow-x: hidden; overflow-y: auto; min-width: max(15%, 120px); padding-top: 0.5em; margin-top: -4px !important; } +.extra-network-subdirs { background: var(--input-background-fill); overflow-x: hidden; overflow-y: auto; min-width: max(15%, 120px); padding-top: 0.5em; } .extra-networks-page { display: flex } .extra-network-cards { display: flex; flex-wrap: wrap; overflow-y: auto; overflow-x: hidden; align-content: flex-start; width: -moz-available; width: -webkit-fill-available; } .extra-network-cards .card { height: fit-content; margin: 0 0 0.5em 0.5em; position: relative; scroll-snap-align: start; scroll-margin-top: 0; } diff --git a/javascript/sdnext.css b/javascript/sdnext.css index 60d835cd4..6e33551ef 100644 --- a/javascript/sdnext.css +++ b/javascript/sdnext.css @@ -14,7 +14,7 @@ table { overflow-x: auto !important; overflow-y: auto !important; } td { border-bottom: none !important; padding: 0 0.5em !important; } tr { border-bottom: none !important; padding: 0 0.5em !important; } td > div > span { overflow-y: auto; max-height: 3em; overflow-x: hidden; } -textarea { overflow-y: auto !important; } +textarea { overflow-y: auto !important; border-radius: 4px !important; } span { font-size: var(--text-md) !important; } button { font-size: var(--text-lg) !important; min-width: unset !important; } input[type='color'] { width: 64px; height: 32px; } @@ -217,7 +217,7 @@ table.settings-value-table td { padding: 0.4em; border: 1px solid #ccc; max-widt .extra_networks_root { width: 0; position: absolute; height: auto; right: 0; top: 13em; z-index: 100; } /* default is sidebar view */ .extra-networks { background: var(--background-color); padding: var(--block-label-padding); } .extra-networks > div { margin: 0; border-bottom: none !important; gap: 0.3em 0; } -.extra-networks .second-line { display: flex; width: -moz-available; width: -webkit-fill-available; gap: 0.3em; box-shadow: var(--input-shadow); } +.extra-networks .second-line { display: flex; width: -moz-available; width: -webkit-fill-available; gap: 0.3em; box-shadow: var(--input-shadow); margin-bottom: 2px; } .extra-networks .search { flex: 1; height: 4em; } .extra-networks .description { flex: 3; } .extra-networks .tab-nav>button { margin-right: 0; height: 24px; padding: 2px 4px 2px 4px; } @@ -226,7 +226,7 @@ table.settings-value-table td { padding: 0.4em; border: 1px solid #ccc; max-widt .extra-networks .custom-button { width: 120px; width: 100%; background: none; justify-content: left; text-align: left; padding: 3px 3px 3px 12px; text-indent: -6px; box-shadow: none; line-break: auto; } .extra-networks .custom-button:hover { background: var(--button-primary-background-fill) } .extra-networks-tab { padding: 0 !important; } -.extra-network-subdirs { background: var(--input-background-fill); overflow-x: hidden; overflow-y: auto; min-width: max(15%, 120px); padding-top: 0.5em; margin-top: -4px !important; } +.extra-network-subdirs { background: var(--input-background-fill); overflow-x: hidden; overflow-y: auto; min-width: max(15%, 120px); padding-top: 0.5em; border-radius: 4px; } .extra-networks-page { display: flex } .extra-network-cards { display: flex; flex-wrap: wrap; overflow-y: auto; overflow-x: hidden; align-content: flex-start; width: -moz-available; width: -webkit-fill-available; } .extra-network-cards .card { height: fit-content; margin: 0 0 0.5em 0.5em; position: relative; scroll-snap-align: start; scroll-margin-top: 0; } From e1f5feffe06caa64531436f57771c1a39bc26e69 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 24 Dec 2024 08:40:46 -0500 Subject: [PATCH 141/249] update requirements and changelog Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 8 ++++---- README.md | 5 ++--- extensions-builtin/stable-diffusion-webui-rembg | 2 +- installer.py | 2 +- modules/processing_vae.py | 5 ++--- requirements.txt | 2 +- 6 files changed, 11 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0dac17552..b778dca21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,8 @@ # Change Log for SD.Next -## Update for 2024-12-23 +## Update for 2024-12-24 -### Highlights for 2024-12-23 +### Highlights for 2024-12-24 ### SD.Next Xmass edition: *What's new?* @@ -28,11 +28,11 @@ And a lot of **Control** and **IPAdapter** goodies Plus couple of new integrated workflows such as [FreeScale](https://github.com/ali-vilab/FreeScale) and [Style Aligned Image Generation](https://style-aligned-gen.github.io/) And it wouldn't be a *Xmass edition* without couple of custom themes: *Snowflake* and *Elf-Green*! -All-in-all, we're around ~160 commits worth of updates, check changelog for full list +All-in-all, we're around ~180 commits worth of updates, check the changelog for full list [ReadMe](https://github.com/vladmandic/automatic/blob/master/README.md) | [ChangeLog](https://github.com/vladmandic/automatic/blob/master/CHANGELOG.md) | [Docs](https://vladmandic.github.io/sdnext-docs/) | [WiKi](https://github.com/vladmandic/automatic/wiki) | [Discord](https://discord.com/invite/sd-next-federal-batch-inspectors-1101998836328697867) -## Details for 2024-12-23 +## Details for 2024-12-24 ### New models and integrations diff --git a/README.md b/README.md index e33b3112a..2a220ca1d 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ All individual features are not listed here, instead check [ChangeLog](CHANGELOG.md) for full list of changes - Multiple UIs! ▹ **Standard | Modern** -- Multiple diffusion models! +- Multiple [diffusion models](https://vladmandic.github.io/sdnext-docs/Model-Support/)! - Built-in Control for Text, Image, Batch and video processing! - Multiplatform! ▹ **Windows | Linux | MacOS | nVidia | AMD | IntelArc/IPEX | DirectML | OpenVINO | ONNX+Olive | ZLUDA** @@ -36,7 +36,6 @@ All individual features are not listed here, instead check [ChangeLog](CHANGELOG - Optimized processing with latest `torch` developments with built-in support for `torch.compile` and multiple compile backends: *Triton, ZLUDA, StableFast, DeepCache, OpenVINO, NNCF, IPEX, OneDiff* - Built-in queue management -- Enterprise level logging and hardened API - Built in installer with automatic updates and dependency management - Mobile compatible @@ -68,7 +67,7 @@ SD.Next supports broad range of models: [supported models](https://vladmandic.gi - Any GPU or device compatible with **OpenVINO** libraries on both *Windows and Linux* - *Apple M1/M2* on *OSX* using built-in support in Torch with **MPS** optimizations - *ONNX/Olive* -- *AMD* GPUs on Windows using **ZLUDA** libraries +- *AMD* GPUs on Windows using **ZLUDA** libraries ## Getting started diff --git a/extensions-builtin/stable-diffusion-webui-rembg b/extensions-builtin/stable-diffusion-webui-rembg index ff2bbd168..50bc931ce 160000 --- a/extensions-builtin/stable-diffusion-webui-rembg +++ b/extensions-builtin/stable-diffusion-webui-rembg @@ -1 +1 @@ -Subproject commit ff2bbd16820617ce26183a6c6538dc95a312b5de +Subproject commit 50bc931ce6715e30bb86da52033e01e98add4e6c diff --git a/installer.py b/installer.py index 46ccce92c..43c4e84e3 100644 --- a/installer.py +++ b/installer.py @@ -459,7 +459,7 @@ def check_python(supported_minors=[9, 10, 11, 12], reason=None): def check_diffusers(): if args.skip_all or args.skip_git: return - sha = '4b557132ce955d58fd84572c03e79f43bdc91450' # diffusers commit hash + sha = '6dfaec348780c6153a4cfd03a01972a291d67f82' # diffusers commit hash pkg = pkg_resources.working_set.by_key.get('diffusers', None) minor = int(pkg.version.split('.')[1] if pkg is not None else 0) cur = opts.get('diffusers_version', '') if minor > 0 else '' diff --git a/modules/processing_vae.py b/modules/processing_vae.py index 1710bd992..95b83daa4 100644 --- a/modules/processing_vae.py +++ b/modules/processing_vae.py @@ -141,7 +141,8 @@ def full_vae_decode(latents, model): log_debug(f'VAE config: {model.vae.config}') try: - decoded = model.vae.decode(latents, return_dict=False)[0] + with devices.inference_context(): + decoded = model.vae.decode(latents, return_dict=False)[0] except Exception as e: shared.log.error(f'VAE decode: {stats} {e}') if 'out of memory' not in str(e): @@ -159,8 +160,6 @@ def full_vae_decode(latents, model): model.vae.apply(sd_models.convert_to_faketensors) devices.torch_gc(force=True) - # if shared.opts.diffusers_offload_mode == "balanced": - # shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) elif shared.opts.diffusers_move_unet and not getattr(model, 'has_accelerate', False) and base_device is not None: sd_models.move_base(model, base_device) t1 = time.time() diff --git a/requirements.txt b/requirements.txt index f44f758e2..9b2a52ffa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -45,7 +45,7 @@ accelerate==1.2.1 opencv-contrib-python-headless==4.9.0.80 einops==0.4.1 gradio==3.43.2 -huggingface_hub==0.26.5 +huggingface_hub==0.27.0 numexpr==2.8.8 numpy==1.26.4 numba==0.59.1 From fe3f54e26f9340b031bcb812ab672d5bed005cc5 Mon Sep 17 00:00:00 2001 From: vladmandic Date: Mon, 9 Dec 2024 08:27:14 +0000 Subject: [PATCH 142/249] =?UTF-8?q?Deploying=20to=20master=20from=20@=20vl?= =?UTF-8?q?admandic/automatic@f79e85efeb5c43a344c9979c9dc5fa4c56a0d8ed=20?= =?UTF-8?q?=F0=9F=9A=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2a220ca1d..2a95373d0 100644 --- a/README.md +++ b/README.md @@ -112,7 +112,7 @@ check [ChangeLog](https://vladmandic.github.io/sdnext-docs/CHANGELOG/) for when ### Sponsors
-Allan GrantBrent OzarMatthew Runoa.v.mantzarisSML (See-ming Lee) +Allan GrantBrent Ozara.v.mantzarisSML (See-ming Lee)

From f1c47c9ceb97b6586cea2460496ce83eab7b714e Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 24 Dec 2024 08:57:36 -0500 Subject: [PATCH 143/249] cleanup Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 2 +- extensions-builtin/sdnext-modernui | 2 +- html/locale_ko.json | 2 +- javascript/black-teal-reimagined.css | 4 ++-- modules/ui_extra_networks.py | 16 ++++++++-------- modules/ui_extra_networks_checkpoints.py | 2 +- wiki | 2 +- 7 files changed, 15 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b778dca21..1721e40d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2287,7 +2287,7 @@ Plus some nifty new modules such as **FaceID** automatic face guidance using emb can remove artifacts and hard edges of inpaint area but also remove some details from original - **chaiNNer** fix `NaN` issues due to autocast - **Upscale** increase limit from 4x to 8x given the quality of some upscalers - - **Extra Networks** fix sort + - **Networks** fix sort - reduced default **CFG scale** from 6 to 4 to be more out-of-the-box compatibile with LCM/Turbo models - disable google fonts check on server startup - fix torchvision/basicsr compatibility diff --git a/extensions-builtin/sdnext-modernui b/extensions-builtin/sdnext-modernui index 3f53ff719..f6f629dfb 160000 --- a/extensions-builtin/sdnext-modernui +++ b/extensions-builtin/sdnext-modernui @@ -1 +1 @@ -Subproject commit 3f53ff719f8024003873dd98bb64f21e9401844c +Subproject commit f6f629dfbe63d8d6ed34f4ab2f9c7938914bf617 diff --git a/html/locale_ko.json b/html/locale_ko.json index 0ddf62afb..3d1dfd9df 100644 --- a/html/locale_ko.json +++ b/html/locale_ko.json @@ -182,7 +182,7 @@ {"id":"","label":"Training","localized":"학습(훈련)","hint":""}, {"id":"","label":"Interrogate","localized":"이미지➠텍스트","hint":""}, {"id":"","label":"Upscaling","localized":"업스케일링","hint":""}, - {"id":"","label":"Extra Networks","localized":"엑스트라 네트워크","hint":""}, + {"id":"","label":"Networks","localized":"엑스트라 네트워크","hint":""}, {"id":"","label":"Licenses","localized":"라이선스 정보","hint":""}, {"id":"","label":"Show all pages","localized":"모든 페이지 보기","hint":""}, {"id":"","label":"Request browser notifications","localized":"브라우저 알림 요청","hint":""} diff --git a/javascript/black-teal-reimagined.css b/javascript/black-teal-reimagined.css index 28176d247..c221d7e4e 100644 --- a/javascript/black-teal-reimagined.css +++ b/javascript/black-teal-reimagined.css @@ -783,7 +783,7 @@ svg.feather.feather-image, margin-top: var(--spacing-lg); } -/* Extra Networks Container */ +/* Networks Container */ #extra_networks_root { z-index: 100; background: var(--background-color); @@ -801,7 +801,7 @@ svg.feather.feather-image, } } -/* Extra Networks Styles */ +/* Networks Styles */ .extra-networks { border-left: 2px solid var(--highlight-color) !important; padding-left: 4px; diff --git a/modules/ui_extra_networks.py b/modules/ui_extra_networks.py index 89cea44b8..6389da194 100644 --- a/modules/ui_extra_networks.py +++ b/modules/ui_extra_networks.py @@ -203,9 +203,9 @@ def create_items(self, tabname): self.refresh_time = time.time() except Exception as e: self.items = [] - shared.log.error(f'Extra networks error listing items: class={self.__class__.__name__} tab={tabname} {e}') + shared.log.error(f'Networks: listing items class={self.__class__.__name__} tab={tabname} {e}') if os.environ.get('SD_EN_DEBUG', None): - errors.display(e, f'Extra networks error listing items: class={self.__class__.__name__} tab={tabname}') + errors.display(e, f'Networks: listing items: class={self.__class__.__name__} tab={tabname}') for item in self.items: if item is None: continue @@ -321,9 +321,9 @@ def random_bright_color(): args['title'] += f'\nAlias: {alias}' return self.card.format(**args) except Exception as e: - shared.log.error(f'Extra networks item error: page={tabname} item={item["name"]} {e}') + shared.log.error(f'Networks: item error: page={tabname} item={item["name"]} {e}') if os.environ.get('SD_EN_DEBUG', None) is not None: - errors.display(e, 'Extra networks') + errors.display(e, 'Networks') return "" def find_preview_file(self, path): @@ -567,7 +567,7 @@ def state_change(state_text): nonlocal state state = SimpleNamespace(**json.loads(state_text)) except Exception as e: - shared.log.error(f'Extra networks state error: {e}') + shared.log.error(f'Networks: state error: {e}') return _page, _item = get_item(state) # shared.log.debug(f'Extra network: op={state.op} page={page.title if page is not None else None} item={item.filename if item is not None else None}') @@ -880,7 +880,7 @@ def ui_refresh_click(title): page.refresh_time = 0 page.refresh() page.create_page(ui.tabname) - shared.log.debug(f"Refreshing Extra networks: page='{page.title}' items={len(page.items)} tab={ui.tabname}") + shared.log.debug(f"Networks: refresh page='{page.title}' items={len(page.items)} tab={ui.tabname}") pages.append(page.html) ui.search.update(title) return pages @@ -894,7 +894,7 @@ def ui_view_cards(title): page.card = card_full if page.view == 'gallery' else card_list page.html = '' page.create_page(ui.tabname) - shared.log.debug(f"Refreshing Extra networks: page='{page.title}' items={len(page.items)} tab={ui.tabname} view={page.view}") + shared.log.debug(f"Networks: refresh page='{page.title}' items={len(page.items)} tab={ui.tabname} view={page.view}") pages.append(page.html) ui.search.update(title) return pages @@ -946,7 +946,7 @@ def ui_sort_cards(sort_order): if shared.opts.extra_networks_sort != sort_order: shared.opts.extra_networks_sort = sort_order shared.opts.save(shared.config_filename) - return f'Extra networks sort={sort_order}' + return f'Networks: sort={sort_order}' dummy = gr.State(value=False) # pylint: disable=abstract-class-instantiated button_parent.click(fn=toggle_visibility, inputs=[ui.visible], outputs=[ui.visible, container, button_parent]) diff --git a/modules/ui_extra_networks_checkpoints.py b/modules/ui_extra_networks_checkpoints.py index 9b66b754e..66dd21cb0 100644 --- a/modules/ui_extra_networks_checkpoints.py +++ b/modules/ui_extra_networks_checkpoints.py @@ -27,7 +27,7 @@ def list_reference(self): # pylint: disable=inconsistent-return-statements experimental = v.get('experimental', False) if experimental: if shared.cmd_opts.experimental: - shared.log.debug(f'Extra networks experimental: model="{k}"') + shared.log.debug(f'Networks: experimental model="{k}"') else: continue preview = v.get('preview', v['path']) diff --git a/wiki b/wiki index 56ba782f7..22951c981 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 56ba782f744bb8f6928f6c365d6ffc547d339548 +Subproject commit 22951c9818e44f3bfeecfd06b1c154230f711ea7 From ef8a0f031aec6717fb4500851731514ba71ef916 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 24 Dec 2024 09:19:22 -0500 Subject: [PATCH 144/249] prototype image-search Signed-off-by: Vladimir Mandic --- cli/image-search.py | 249 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 249 insertions(+) create mode 100644 cli/image-search.py diff --git a/cli/image-search.py b/cli/image-search.py new file mode 100644 index 000000000..3115a1fd9 --- /dev/null +++ b/cli/image-search.py @@ -0,0 +1,249 @@ +from typing import Union +import os +import re +import logging +from tqdm.rich import tqdm +import torch +import PIL +import faiss +import numpy as np +import pandas as pd +import transformers + + +class ImageDB: + # TODO index: quantize and train faiss index + # TODO index: clip batch processing + def __init__(self, + name:str='db', + fmt:str='json', + cache_dir:str=None, + dtype:torch.dtype=torch.float16, + device:torch.device=torch.device('cpu'), + model:str='openai/clip-vit-large-patch14', # 'facebook/dinov2-small' + debug:bool=False, + pbar:bool=True, + ): + self.format = fmt + self.name = name + self.cache_dir = cache_dir + self.processor: transformers.AutoImageProcessor = None + self.model: transformers.AutoModel = None + self.tokenizer = transformers.AutoTokenizer = None + self.device: torch.device = device + self.dtype: torch.dtype = dtype + self.dimension = 768 if 'clip' in model else 384 + self.debug = debug + self.pbar = pbar + self.repo = model + self.df = pd.DataFrame([], columns=['filename', 'timestamp', 'metadata']) # image/metadata database + self.index = faiss.IndexFlatL2(self.dimension) # embed database + self.log = logging.getLogger(__name__) + self.err = logging.getLogger(__name__).error + self.log = logging.getLogger(__name__).info if self.debug else logging.getLogger(__name__).debug + # self.init() + # self.load() + + def __str__(self): + return f'db: name="{self.name}" format={self.format} device={self.device} dtype={self.dtype} dimension={self.dimension} model="{self.repo}" records={len(self.df)} index={self.index.ntotal}' + + def init(self): # initialize models + if self.processor is None or self.model is None: + if 'clip' in self.repo: + self.processor = transformers.CLIPImageProcessor.from_pretrained(self.repo, cache_dir=self.cache_dir) + self.tokenizer = transformers.CLIPTokenizer.from_pretrained(self.repo, cache_dir=self.cache_dir) + self.model = transformers.CLIPModel.from_pretrained(self.repo, cache_dir=self.cache_dir).to(device=self.device, dtype=self.dtype) + elif 'dino' in self.repo: + self.processor = transformers.AutoImageProcessor.from_pretrained(self.repo, cache_dir=self.cache_dir) + self.model = transformers.AutoModel.from_pretrained(self.repo, cache_dir=self.cache_dir).to(device=self.device, dtype=self.dtype) + else: + self.err(f'db: model="{self.repo}" unknown') + self.log(f'db: load model="{self.repo}" cache="{self.cache_dir}" device={self.device} dtype={self.dtype}') + + def load(self): # load db to disk + if self.format == 'json' and os.path.exists(f'{self.name}.json'): + self.df = pd.read_json(f'{self.name}.json') + elif self.format == 'csv' and os.path.exists(f'{self.name}.csv'): + self.df = pd.read_csv(f'{self.name}.csv') + elif self.format == 'pickle' and os.path.exists(f'{self.name}.pkl'): + self.df = pd.read_pickle(f'{self.name}.parquet') + if os.path.exists(f'{self.name}.index'): + self.index = faiss.read_index(f'{self.name}.index') + if self.index.ntotal != len(self.df): + self.err(f'db: index={self.index.ntotal} data={len(self.df)} mismatch') + self.index = faiss.IndexFlatL2(self.dimension) + self.df = pd.DataFrame([], columns=['filename', 'timestamp', 'metadata']) + self.log(f'db: load data={len(self.df)} name={self.name} format={self.format} name={self.name}') + + def save(self): # save db to disk + if self.format == 'json': + self.df.to_json(f'{self.name}.json') + elif self.format == 'csv': + self.df.to_csv(f'{self.name}.csv') + elif self.format == 'pickle': + self.df.to_pickle(f'{self.name}.pkl') + faiss.write_index(self.index, f'{self.name}.index') + self.log(f'db: save data={len(self.df)} name={self.name} format={self.format} name={self.name}') + + def normalize(self, embed) -> np.ndarray: # normalize embed before using it + embed = embed.detach().float().cpu().numpy() + faiss.normalize_L2(embed) + return embed + + def embedding(self, query: Union[PIL.Image.Image | str]) -> np.ndarray: # calculate embed for prompt or image + if self.processor is None or self.model is None: + self.err('db: model not loaded') + if isinstance(query, str) and os.path.exists(query): + query = PIL.Image.open(query).convert('RGB') + self.model = self.model.to(self.device) + with torch.no_grad(): + if 'clip' in self.repo: + if isinstance(query, str): + processed = self.tokenizer(text=query, padding=True, return_tensors="pt").to(device=self.device) + results = self.model.get_text_features(**processed) + else: + processed = self.processor(images=query, return_tensors="pt").to(device=self.device, dtype=self.dtype) + results = self.model.get_image_features(**processed) + elif 'dino' in self.repo: + processed = self.processor(images=query, return_tensors="pt").to(device=self.device, dtype=self.dtype) + results = self.model(**processed) + results = results.last_hidden_state.mean(dim=1) + else: + self.err(f'db: model="{self.repo}" unknown') + return None + return self.normalize(results) + + def add(self, embed, filename=None, metadata=None): # add embed to db + rec = pd.DataFrame([{'filename': filename, 'timestamp': pd.Timestamp.now(), 'metadata': metadata}]) + if len(self.df) > 0: + self.df = pd.concat([self.df, rec], ignore_index=True) + else: + self.df = rec + self.index.add(embed) + + def search(self, filename: str = None, metadata: str = None, embed: np.ndarray = None, k=10, d=1.0): # search by filename/metadata/prompt-embed/image-embed + def dct(record: pd.DataFrame, mode: str, distance: float = None): + if distance is not None: + return {'type': mode, 'filename': record[1]['filename'], 'metadata': record[1]['metadata'], 'distance': round(distance, 2)} + else: + return {'type': mode, 'filename': record[1]['filename'], 'metadata': record[1]['metadata']} + + if self.index.ntotal == 0: + return + self.log(f'db: search k={k} d={d}') + if embed is not None: + distances, indexes = self.index.search(embed, k) + records = self.df.iloc[indexes[0]] + for record, distance in zip(records.iterrows(), distances[0]): + if d <= 0 or distance <= d: + yield dct(record, distance=distance, mode='embed') + if filename is not None: + records = self.df[self.df['filename'].str.contains(filename, na=False, case=False)] + for record in records.iterrows(): + yield dct(record, mode='filename') + if metadata is not None: + records = self.df[self.df['metadata'].str.contains(filename, na=False, case=False)] + for record in records.iterrows(): + yield dct(record, mode='metadata') + + def decode(self, s: bytes): # decode byte-encoded exif metadata + remove_prefix = lambda text, prefix: text[len(prefix):] if text.startswith(prefix) else text # pylint: disable=unnecessary-lambda-assignment + for encoding in ['utf-8', 'utf-16', 'ascii', 'latin_1', 'cp1252', 'cp437']: # try different encodings + try: + s = remove_prefix(s, b'UNICODE') + s = remove_prefix(s, b'ASCII') + s = remove_prefix(s, b'\x00') + val = s.decode(encoding, errors="strict") + val = re.sub(r'[\x00-\x09\n\s\s+]', '', val).strip() # remove remaining special characters, new line breaks, and double empty spaces + if len(val) == 0: # remove empty strings + val = None + return val + except Exception: + pass + return None + + def metadata(self, image: PIL.Image.Image): # get exif metadata from image + exif = image._getexif() # pylint: disable=protected-access + if exif is None: + return '' + for k, v in exif.items(): + if k == 37510: # comment + return self.decode(v) + return '' + + def image(self, filename: str, image=None): # add file/image to db + try: + if image is None: + image = PIL.Image.open(filename) + image.load() + embed = self.embedding(image.convert('RGB')) + metadata = self.metadata(image) + image.close() + self.add(embed, filename=filename, metadata=metadata) + except Exception as _e: + # self.err(f'db: {str(_e)}') + pass + + def folder(self, folder: str): # add all files from folder to db + files = [] + for root, _subdir, _files in os.walk(folder): + for f in _files: + files.append(os.path.join(root, f)) + if self.pbar: + for f in tqdm(files): + self.image(filename=f) + else: + for f in files: + self.image(filename=f) + + def offload(self): # offload model to cpu + if self.model is not None: + self.model = self.model.to('cpu') + + +if __name__ == '__main__': + import time + import argparse + logging.basicConfig(level=logging.INFO) + parser = argparse.ArgumentParser(description = 'image-search') + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument('--search', action='store_true', help='run search') + group.add_argument('--index', action='store_true', help='run indexing') + parser.add_argument('--db', default='db', help='database name') + parser.add_argument('--model', default='openai/clip-vit-large-patch14', help='huggingface model') + parser.add_argument('--cache', default='/mnt/models/huggingface', help='cache folder') + parser.add_argument('input', nargs='*', default=os.getcwd()) + args = parser.parse_args() + + db = ImageDB( + name=args.db, + model=args.model, # 'facebook/dinov2-small' + cache_dir=args.cache, + dtype=torch.bfloat16, + device=torch.device('cuda'), + debug=True, + pbar=True, + ) + db.init() + db.load() + print(db) + + if args.index: + t0 = time.time() + if len(args.input) > 0: + for fn in args.input: + if os.path.isfile(fn): + db.image(filename=fn) + elif os.path.isdir(fn): + db.folder(folder=fn) + t1 = time.time() + print('index', t1-t0) + db.save() + db.offload() + + if args.search: + for ref in args.input: + emb = db.embedding(ref) + res = db.search(filename=ref, metadata=ref, embed=emb, k=10, d=0) + for r in res: + print(ref, r) From f895180e844c6e9347f84a064d25232ecc0c8947 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 24 Dec 2024 16:09:25 -0500 Subject: [PATCH 145/249] remove experimental from consistory Signed-off-by: Vladimir Mandic --- modules/lora/networks.py | 2 ++ scripts/consistory_ext.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/lora/networks.py b/modules/lora/networks.py index fc90ddd2d..edb826080 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -311,6 +311,8 @@ def network_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.n t0 = time.time() weights_backup = getattr(self, "network_weights_backup", None) + if (shared.opts.lora_fuse_diffusers and not isinstance(weights_backup, bool)) or (not shared.opts.lora_fuse_diffusers and isinstance(weights_backup, bool)): + weights_backup = None # invalidate so we can change direct/backup on-the-fly if weights_backup is None and wanted_names != (): # pylint: disable=C1803 weight = getattr(self, 'weight', None) self.network_weights_backup = None diff --git a/scripts/consistory_ext.py b/scripts/consistory_ext.py index 45de1ea6c..7a8f21e3b 100644 --- a/scripts/consistory_ext.py +++ b/scripts/consistory_ext.py @@ -25,7 +25,7 @@ def title(self): return 'ConsiStory: Consistent Image Generation' def show(self, is_img2img): - return not is_img2img if shared.native and shared.cmd_opts.experimental else False + return not is_img2img if shared.native else False def reset(self): self.anchor_cache_first_stage = None From f4cb8a1454e5d0b6914725aa73353e3810436aef Mon Sep 17 00:00:00 2001 From: Disty0 Date: Wed, 25 Dec 2024 18:19:43 +0300 Subject: [PATCH 146/249] Better live previews --- modules/processing_callbacks.py | 18 ++++++++++++++++++ modules/sd_models.py | 15 +++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/modules/processing_callbacks.py b/modules/processing_callbacks.py index 0b4c7dfe1..c05529ee9 100644 --- a/modules/processing_callbacks.py +++ b/modules/processing_callbacks.py @@ -105,6 +105,24 @@ def diffusers_callback(pipe, step: int = 0, timestep: int = 0, kwargs: dict = {} shared.state.current_latent = pipe._unpack_latents(kwargs['latents'], height, width, pipe.vae_scale_factor) # pylint: disable=protected-access else: shared.state.current_latent = kwargs['latents'] + + if hasattr(pipe, "scheduler") and hasattr(pipe.scheduler, "sigmas"): + noise_pred = None + if kwargs.get("noise_pred", None) is not None: + noise_pred = kwargs.get("noise_pred") + elif kwargs.get("predicted_image_embedding", None) is not None: + noise_pred = kwargs.get("predicted_image_embedding") + if noise_pred is not None: + sigma = pipe.scheduler.sigmas[step] + sigma_next = pipe.scheduler.sigmas[step + 1] + original_sample = shared.state.current_latent - (noise_pred * (sigma_next-sigma)) + if "flow" in pipe.scheduler.__class__.__name__.lower(): + shared.state.current_latent = original_sample - (noise_pred * sigma) + elif hasattr(pipe.scheduler, "config") and hasattr(pipe.scheduler.config, "prediction_type"): + if pipe.scheduler.config.prediction_type == "epsilon": + shared.state.current_latent = original_sample - (noise_pred * sigma) + elif pipe.scheduler.config.prediction_type == "v_prediction": + shared.state.current_latent = noise_pred * (-sigma / (sigma**2 + 1) ** 0.5) + (original_sample / (sigma**2 + 1)) except Exception as e: shared.log.error(f'Callback: {e}') if shared.cmd_opts.profile and shared.profiler is not None: diff --git a/modules/sd_models.py b/modules/sd_models.py index 087e8ecfc..fa6398d3e 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -221,6 +221,7 @@ def copy_diffuser_options(new_pipe, orig_pipe): new_pipe.is_sdxl = getattr(orig_pipe, 'is_sdxl', False) # a1111 compatibility item new_pipe.is_sd2 = getattr(orig_pipe, 'is_sd2', False) new_pipe.is_sd1 = getattr(orig_pipe, 'is_sd1', True) + add_noise_pred_to_diffusers_callback(new_pipe) if new_pipe.has_accelerate: set_accelerate(new_pipe) @@ -975,6 +976,8 @@ def load_diffuser(checkpoint_info=None, already_loaded_state_dict=None, timer=No if model_type not in ['Stable Cascade']: # need a special-case sd_unet.load_unet(sd_model) + add_noise_pred_to_diffusers_callback(sd_model) + timer.record("load") if op == 'refiner': @@ -1276,9 +1279,11 @@ def set_diffuser_pipe(pipe, new_pipe_type): new_pipe.is_sd1 = getattr(pipe, 'is_sd1', True) if hasattr(new_pipe, 'watermark'): new_pipe.watermark = NoWatermark() + add_noise_pred_to_diffusers_callback(new_pipe) if hasattr(new_pipe, 'pipe'): # also handle nested pipelines new_pipe.pipe = set_diffuser_pipe(new_pipe.pipe, new_pipe_type) + add_noise_pred_to_diffusers_callback(new_pipe.pipe) fn = f'{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access shared.log.debug(f"Pipeline class change: original={cls} target={new_pipe.__class__.__name__} device={pipe.device} fn={fn}") # pylint: disable=protected-access @@ -1333,6 +1338,16 @@ def set_attn(pipe, attention): pipe.current_attn_name = shared.opts.cross_attention_optimization +def add_noise_pred_to_diffusers_callback(pipe): + if pipe.__class__.__name__.startswith("StableDiffusion"): + pipe._callback_tensor_inputs.append("noise_pred") + elif pipe.__class__.__name__.startswith("StableCascade"): + pipe.prior_pipe._callback_tensor_inputs.append("predicted_image_embedding") + elif hasattr(pipe, "scheduler") and "flow" in pipe.scheduler.__class__.__name__.lower(): + pipe._callback_tensor_inputs.append("noise_pred") + return pipe + + def get_native(pipe: diffusers.DiffusionPipeline): if hasattr(pipe, "vae") and hasattr(pipe.vae.config, "sample_size"): # Stable Diffusion From 8f503e58d13d69fdcdd73f4cfc4a770235530acc Mon Sep 17 00:00:00 2001 From: Disty0 Date: Wed, 25 Dec 2024 18:43:10 +0300 Subject: [PATCH 147/249] Live previews add flow_prediction --- modules/processing_callbacks.py | 2 +- modules/sd_models.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/processing_callbacks.py b/modules/processing_callbacks.py index c05529ee9..b896893f4 100644 --- a/modules/processing_callbacks.py +++ b/modules/processing_callbacks.py @@ -119,7 +119,7 @@ def diffusers_callback(pipe, step: int = 0, timestep: int = 0, kwargs: dict = {} if "flow" in pipe.scheduler.__class__.__name__.lower(): shared.state.current_latent = original_sample - (noise_pred * sigma) elif hasattr(pipe.scheduler, "config") and hasattr(pipe.scheduler.config, "prediction_type"): - if pipe.scheduler.config.prediction_type == "epsilon": + if pipe.scheduler.config.prediction_type in {"epsilon", "flow_prediction"}: shared.state.current_latent = original_sample - (noise_pred * sigma) elif pipe.scheduler.config.prediction_type == "v_prediction": shared.state.current_latent = noise_pred * (-sigma / (sigma**2 + 1) ** 0.5) + (original_sample / (sigma**2 + 1)) diff --git a/modules/sd_models.py b/modules/sd_models.py index fa6398d3e..a52e15e7e 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -1345,6 +1345,8 @@ def add_noise_pred_to_diffusers_callback(pipe): pipe.prior_pipe._callback_tensor_inputs.append("predicted_image_embedding") elif hasattr(pipe, "scheduler") and "flow" in pipe.scheduler.__class__.__name__.lower(): pipe._callback_tensor_inputs.append("noise_pred") + elif hasattr(pipe, "default_scheduler") and "flow" in pipe.default_scheduler.__class__.__name__.lower(): + pipe._callback_tensor_inputs.append("noise_pred") return pipe From 4ab960330c28f5ff21c9a400e21b3bde10c1970e Mon Sep 17 00:00:00 2001 From: Disty0 Date: Wed, 25 Dec 2024 20:06:20 +0300 Subject: [PATCH 148/249] Move simga calcs to do_set_current_image --- modules/processing_callbacks.py | 21 +++++---------------- modules/sd_samplers.py | 8 ++++++++ modules/shared_state.py | 17 ++++++++++++++++- 3 files changed, 29 insertions(+), 17 deletions(-) diff --git a/modules/processing_callbacks.py b/modules/processing_callbacks.py index b896893f4..39a6c66d8 100644 --- a/modules/processing_callbacks.py +++ b/modules/processing_callbacks.py @@ -106,23 +106,12 @@ def diffusers_callback(pipe, step: int = 0, timestep: int = 0, kwargs: dict = {} else: shared.state.current_latent = kwargs['latents'] + shared.state.current_noise_pred = kwargs.get("noise_pred", None) + if shared.state.current_noise_pred is None: + shared.state.current_noise_pred = kwargs.get("predicted_image_embedding", None) if hasattr(pipe, "scheduler") and hasattr(pipe.scheduler, "sigmas"): - noise_pred = None - if kwargs.get("noise_pred", None) is not None: - noise_pred = kwargs.get("noise_pred") - elif kwargs.get("predicted_image_embedding", None) is not None: - noise_pred = kwargs.get("predicted_image_embedding") - if noise_pred is not None: - sigma = pipe.scheduler.sigmas[step] - sigma_next = pipe.scheduler.sigmas[step + 1] - original_sample = shared.state.current_latent - (noise_pred * (sigma_next-sigma)) - if "flow" in pipe.scheduler.__class__.__name__.lower(): - shared.state.current_latent = original_sample - (noise_pred * sigma) - elif hasattr(pipe.scheduler, "config") and hasattr(pipe.scheduler.config, "prediction_type"): - if pipe.scheduler.config.prediction_type in {"epsilon", "flow_prediction"}: - shared.state.current_latent = original_sample - (noise_pred * sigma) - elif pipe.scheduler.config.prediction_type == "v_prediction": - shared.state.current_latent = noise_pred * (-sigma / (sigma**2 + 1) ** 0.5) + (original_sample / (sigma**2 + 1)) + shared.state.current_sigma = pipe.scheduler.sigmas[step] + shared.state.current_sigma_next = pipe.scheduler.sigmas[step + 1] except Exception as e: shared.log.error(f'Callback: {e}') if shared.cmd_opts.profile and shared.profiler is not None: diff --git a/modules/sd_samplers.py b/modules/sd_samplers.py index dc58a2419..4d1666ff0 100644 --- a/modules/sd_samplers.py +++ b/modules/sd_samplers.py @@ -62,6 +62,10 @@ def create_sampler(name, model): model.prior_pipe.scheduler.config.clip_sample = False config = {k: v for k, v in model.scheduler.config.items() if not k.startswith('_')} shared.log.debug(f'Sampler: sampler=default class={current}: {config}') + if "flow" in model.scheduler.__class__.__name__.lower(): + shared.state.prediction_type = "flow_prediction" + elif hasattr(model.scheduler, "config") and hasattr(model.scheduler.config, "prediction_type"): + shared.state.prediction_type = model.scheduler.config.prediction_type return model.scheduler config = find_sampler_config(name) if config is None or config.constructor is None: @@ -94,6 +98,10 @@ def create_sampler(name, model): if hasattr(model, "prior_pipe") and hasattr(model.prior_pipe, "scheduler"): model.prior_pipe.scheduler = sampler.sampler model.prior_pipe.scheduler.config.clip_sample = False + if "flow" in model.scheduler.__class__.__name__.lower(): + shared.state.prediction_type = "flow_prediction" + elif hasattr(model.scheduler, "config") and hasattr(model.scheduler.config, "prediction_type"): + shared.state.prediction_type = model.scheduler.config.prediction_type clean_config = {k: v for k, v in sampler.config.items() if v is not None and v is not False} shared.log.debug(f'Sampler: sampler="{sampler.name}" class="{model.scheduler.__class__.__name__} config={clean_config}') return sampler.sampler diff --git a/modules/shared_state.py b/modules/shared_state.py index a3312ec33..2bcf8e23a 100644 --- a/modules/shared_state.py +++ b/modules/shared_state.py @@ -17,10 +17,14 @@ class State: sampling_step = 0 sampling_steps = 0 current_latent = None + current_noise_pred = None + current_sigma = None + current_sigma_next = None current_image = None current_image_sampling_step = 0 id_live_preview = 0 textinfo = None + prediction_type = "epsilon" api = False time_start = None need_restart = False @@ -102,6 +106,9 @@ def begin(self, title="", api=None): self.current_image = None self.current_image_sampling_step = 0 self.current_latent = None + self.current_noise_pred = None + self.current_sigma = None + self.current_sigma_next = None self.id_live_preview = 0 self.interrupted = False self.job = title @@ -113,6 +120,7 @@ def begin(self, title="", api=None): self.sampling_step = 0 self.skipped = False self.textinfo = None + self.prediction_type = "epsilon" self.api = api if api is not None else self.api self.time_start = time.time() if self.debug_output: @@ -152,7 +160,14 @@ def do_set_current_image(self): from modules.shared import opts import modules.sd_samplers # pylint: disable=W0621 try: - image = modules.sd_samplers.samples_to_image_grid(self.current_latent) if opts.show_progress_grid else modules.sd_samplers.sample_to_image(self.current_latent) + sample = self.current_latent + if self.current_noise_pred is not None and self.current_sigma is not None and self.current_sigma_next is not None: + original_sample = sample - (self.current_noise_pred * (self.current_sigma_next-self.current_sigma)) + if self.prediction_type in {"epsilon", "flow_prediction"}: + sample = original_sample - (self.current_noise_pred * self.current_sigma) + elif self.prediction_type == "v_prediction": + sample = self.current_noise_pred * (-self.current_sigma / (self.current_sigma**2 + 1) ** 0.5) + (original_sample / (self.current_sigma**2 + 1)) + image = modules.sd_samplers.samples_to_image_grid(sample) if opts.show_progress_grid else modules.sd_samplers.sample_to_image(sample) self.assign_current_image(image) self.current_image_sampling_step = self.sampling_step except Exception: From f6edbaa0aed9aec0ab0644a24f88165c48913ac9 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Wed, 25 Dec 2024 20:37:16 +0300 Subject: [PATCH 149/249] Live previews add a check for txt2img --- modules/shared_state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/shared_state.py b/modules/shared_state.py index 2bcf8e23a..3920962ed 100644 --- a/modules/shared_state.py +++ b/modules/shared_state.py @@ -161,7 +161,7 @@ def do_set_current_image(self): import modules.sd_samplers # pylint: disable=W0621 try: sample = self.current_latent - if self.current_noise_pred is not None and self.current_sigma is not None and self.current_sigma_next is not None: + if self.job == "txt2img" and self.current_noise_pred is not None and self.current_sigma is not None and self.current_sigma_next is not None: original_sample = sample - (self.current_noise_pred * (self.current_sigma_next-self.current_sigma)) if self.prediction_type in {"epsilon", "flow_prediction"}: sample = original_sample - (self.current_noise_pred * self.current_sigma) From e746f9bcfd81563b99b7d582b3561e2b09c0deff Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 25 Dec 2024 12:38:18 -0500 Subject: [PATCH 150/249] add granular vae tiling options Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 8 +++++ .../Lora/extra_networks_lora.py | 2 +- extensions-builtin/Lora/networks.py | 6 ++-- .../Lora/scripts/lora_script.py | 3 +- modules/extensions.py | 2 +- modules/extra_networks.py | 2 +- modules/lora/networks.py | 21 +++++++++---- modules/sd_models.py | 30 ++++++++++++------- modules/shared.py | 11 +++++-- webui.py | 2 +- wiki | 2 +- 11 files changed, 61 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1721e40d5..414b964d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Change Log for SD.Next +## Update for 2024-12-25 + +### Post release + +- Add granular VAE tiling options in *settings -> variable auto encoder* +- Add legacy option to use old LoRA loader in *settings -> networks* +- Add sigma calculation to VAE preview, thanks @Disty0 + ## Update for 2024-12-24 ### Highlights for 2024-12-24 diff --git a/extensions-builtin/Lora/extra_networks_lora.py b/extensions-builtin/Lora/extra_networks_lora.py index 307d8cc13..76d490eda 100644 --- a/extensions-builtin/Lora/extra_networks_lora.py +++ b/extensions-builtin/Lora/extra_networks_lora.py @@ -129,7 +129,7 @@ def activate(self, p, params_list, step=0): if len(networks.loaded_networks) > 0 and step == 0: self.infotext(p) self.prompt(p) - shared.log.info(f'Load network: type=LoRA apply={[n.name for n in networks.loaded_networks]} te={te_multipliers} unet={unet_multipliers} dims={dyn_dims} load={t1-t0:.2f}') + shared.log.info(f'Load network: type=LoRA apply={[n.name for n in networks.loaded_networks]} method=legacy te={te_multipliers} unet={unet_multipliers} dims={dyn_dims} load={t1-t0:.2f}') def deactivate(self, p): t0 = time.time() diff --git a/extensions-builtin/Lora/networks.py b/extensions-builtin/Lora/networks.py index fd6287c62..1f02f3846 100644 --- a/extensions-builtin/Lora/networks.py +++ b/extensions-builtin/Lora/networks.py @@ -182,11 +182,11 @@ def load_network(name, network_on_disk) -> network.Network: else: net.modules[key] = net_module if len(keys_failed_to_match) > 0: - shared.log.warning(f'LoRA name="{name}" type={set(network_types)} unmatched={len(keys_failed_to_match)} matched={len(matched_networks)}') + shared.log.warning(f'Load network: type=LoRA name="{name}" type={set(network_types)} unmatched={len(keys_failed_to_match)} matched={len(matched_networks)}') if debug: - shared.log.debug(f'LoRA name="{name}" unmatched={keys_failed_to_match}') + shared.log.debug(f'Load network: type=LoRA name="{name}" unmatched={keys_failed_to_match}') else: - shared.log.debug(f'LoRA name="{name}" type={set(network_types)} keys={len(matched_networks)}') + shared.log.debug(f'Load network: type=LoRA name="{name}" type={set(network_types)} keys={len(matched_networks)}') if len(matched_networks) == 0: return None lora_cache[name] = net diff --git a/extensions-builtin/Lora/scripts/lora_script.py b/extensions-builtin/Lora/scripts/lora_script.py index 24723dd7f..5e833aced 100644 --- a/extensions-builtin/Lora/scripts/lora_script.py +++ b/extensions-builtin/Lora/scripts/lora_script.py @@ -57,7 +57,8 @@ def network_replacement(m): d["Prompt"] = re.sub(re_lora, network_replacement, d["Prompt"]) -if not shared.native: +if shared.opts.lora_legacy: + shared.log.debug('Register network: type=LoRA method=legacy') script_callbacks.on_app_started(api_networks) script_callbacks.on_before_ui(before_ui) script_callbacks.on_model_loaded(networks.assign_network_names_to_compvis_modules) diff --git a/modules/extensions.py b/modules/extensions.py index ccd92dbf0..c2e8dceb7 100644 --- a/modules/extensions.py +++ b/modules/extensions.py @@ -154,4 +154,4 @@ def list_extensions(): for dirname, path, is_builtin in extension_paths: extension = Extension(name=dirname, path=path, enabled=dirname not in disabled_extensions, is_builtin=is_builtin) extensions.append(extension) - shared.log.debug(f'Disabled extensions: {[e.name for e in extensions if not e.enabled]}') + shared.log.debug(f'Extensions disabled: {[e.name for e in extensions if not e.enabled]}') diff --git a/modules/extra_networks.py b/modules/extra_networks.py index fe141cca1..8bd742ce0 100644 --- a/modules/extra_networks.py +++ b/modules/extra_networks.py @@ -18,7 +18,7 @@ def register_extra_network(extra_network): def register_default_extra_networks(): from modules.ui_extra_networks_styles import ExtraNetworkStyles register_extra_network(ExtraNetworkStyles()) - if shared.native: + if not shared.opts.lora_legacy: from modules.lora.networks import extra_network_lora register_extra_network(extra_network_lora) if shared.opts.hypernetwork_enabled: diff --git a/modules/lora/networks.py b/modules/lora/networks.py index edb826080..9103fee37 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -131,11 +131,11 @@ def load_safetensors(name, network_on_disk) -> Union[network.Network, None]: else: net.modules[key] = net_module if len(keys_failed_to_match) > 0: - shared.log.warning(f'LoRA name="{name}" type={set(network_types)} unmatched={len(keys_failed_to_match)} matched={len(matched_networks)}') + shared.log.warning(f'Load network: type=LoRA name="{name}" type={set(network_types)} unmatched={len(keys_failed_to_match)} matched={len(matched_networks)}') if debug: - shared.log.debug(f'LoRA name="{name}" unmatched={keys_failed_to_match}') + shared.log.debug(f'Load network: type=LoRA name="{name}" unmatched={keys_failed_to_match}') else: - shared.log.debug(f'LoRA name="{name}" type={set(network_types)} keys={len(matched_networks)} direct={shared.opts.lora_fuse_diffusers}') + shared.log.debug(f'Load network: type=LoRA name="{name}" type={set(network_types)} keys={len(matched_networks)} direct={shared.opts.lora_fuse_diffusers}') if len(matched_networks) == 0: return None lora_cache[name] = net @@ -311,8 +311,14 @@ def network_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.n t0 = time.time() weights_backup = getattr(self, "network_weights_backup", None) - if (shared.opts.lora_fuse_diffusers and not isinstance(weights_backup, bool)) or (not shared.opts.lora_fuse_diffusers and isinstance(weights_backup, bool)): - weights_backup = None # invalidate so we can change direct/backup on-the-fly + bias_backup = getattr(self, "network_bias_backup", None) + if weights_backup is not None or bias_backup is not None: + if (shared.opts.lora_fuse_diffusers and not isinstance(weights_backup, bool)) or (not shared.opts.lora_fuse_diffusers and isinstance(weights_backup, bool)): # invalidate so we can change direct/backup on-the-fly + weights_backup = None + bias_backup = None + self.network_weights_backup = weights_backup + self.network_bias_backup = bias_backup + if weights_backup is None and wanted_names != (): # pylint: disable=C1803 weight = getattr(self, 'weight', None) self.network_weights_backup = None @@ -340,7 +346,6 @@ def network_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.n else: self.network_weights_backup = weight.clone().to(devices.cpu) - bias_backup = getattr(self, "network_bias_backup", None) if bias_backup is None: if getattr(self, 'bias', None) is not None: if shared.opts.lora_fuse_diffusers: @@ -407,6 +412,10 @@ def network_calc_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn. def network_apply_direct(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv], updown: torch.Tensor, ex_bias: torch.Tensor, deactivate: bool = False): weights_backup = getattr(self, "network_weights_backup", False) bias_backup = getattr(self, "network_bias_backup", False) + if not isinstance(weights_backup, bool): + weights_backup = True + if not isinstance(bias_backup, bool): + bias_backup = True if not weights_backup and not bias_backup: return None, None t0 = time.time() diff --git a/modules/sd_models.py b/modules/sd_models.py index a52e15e7e..184a242af 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -226,15 +226,7 @@ def copy_diffuser_options(new_pipe, orig_pipe): set_accelerate(new_pipe) -def set_diffuser_options(sd_model, vae = None, op: str = 'model', offload=True): - if sd_model is None: - shared.log.warning(f'{op} is not loaded') - return - - if hasattr(sd_model, "watermark"): - sd_model.watermark = NoWatermark() - if not (hasattr(sd_model, "has_accelerate") and sd_model.has_accelerate): - sd_model.has_accelerate = False +def set_vae_options(sd_model, vae = None, op: str = 'model'): if hasattr(sd_model, "vae"): if vae is not None: sd_model.vae = vae @@ -254,7 +246,13 @@ def set_diffuser_options(sd_model, vae = None, op: str = 'model', offload=True): sd_model.disable_vae_slicing() if hasattr(sd_model, "enable_vae_tiling"): if shared.opts.diffusers_vae_tiling: - shared.log.debug(f'Setting {op}: component=VAE tiling=True') + if hasattr(sd_model, 'vae') and hasattr(sd_model.vae, 'config') and hasattr(sd_model.vae.config, 'sample_size') and isinstance(sd_model.vae.config.sample_size, int): + sd_model.vae.tile_sample_min_size = int(shared.opts.diffusers_vae_tile_size) + sd_model.vae.tile_latent_min_size = int(sd_model.vae.config.sample_size / (2 ** (len(sd_model.vae.config.block_out_channels) - 1))) + sd_model.vae.tile_overlap_factor = float(shared.opts.diffusers_vae_tile_overlap) + shared.log.debug(f'Setting {op}: component=VAE tiling=True tile={sd_model.vae.tile_sample_min_size} overlap={sd_model.vae.tile_overlap_factor}') + else: + shared.log.debug(f'Setting {op}: component=VAE tiling=True') sd_model.enable_vae_tiling() else: sd_model.disable_vae_tiling() @@ -262,6 +260,18 @@ def set_diffuser_options(sd_model, vae = None, op: str = 'model', offload=True): shared.log.debug(f'Setting {op}: component=VQVAE upcast=True') sd_model.vqvae.to(torch.float32) # vqvae is producing nans in fp16 + +def set_diffuser_options(sd_model, vae = None, op: str = 'model', offload=True): + if sd_model is None: + shared.log.warning(f'{op} is not loaded') + return + + if hasattr(sd_model, "watermark"): + sd_model.watermark = NoWatermark() + if not (hasattr(sd_model, "has_accelerate") and sd_model.has_accelerate): + sd_model.has_accelerate = False + + set_vae_options(sd_model, vae, op) set_diffusers_attention(sd_model) if shared.opts.diffusers_fuse_projections and hasattr(sd_model, 'fuse_qkv_projections'): diff --git a/modules/shared.py b/modules/shared.py index c341deb69..7bbb62e29 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -366,7 +366,7 @@ def list_samplers(): def temp_disable_extensions(): disable_safe = ['sd-webui-controlnet', 'multidiffusion-upscaler-for-automatic1111', 'a1111-sd-webui-lycoris', 'sd-webui-agent-scheduler', 'clip-interrogator-ext', 'stable-diffusion-webui-rembg', 'sd-extension-chainner', 'stable-diffusion-webui-images-browser'] - disable_diffusers = ['sd-webui-controlnet', 'multidiffusion-upscaler-for-automatic1111', 'a1111-sd-webui-lycoris', 'sd-webui-animatediff', 'Lora'] + disable_diffusers = ['sd-webui-controlnet', 'multidiffusion-upscaler-for-automatic1111', 'a1111-sd-webui-lycoris', 'sd-webui-animatediff'] disable_themes = ['sd-webui-lobe-theme', 'cozy-nest', 'sdnext-modernui'] disable_original = [] disabled = [] @@ -422,6 +422,8 @@ def temp_disable_extensions(): for ext in disable_original: if ext.lower() not in opts.disabled_extensions: disabled.append(ext) + if not opts.lora_legacy: + disabled.append('Lora') cmd_opts.controlnet_loglevel = 'WARNING' return disabled @@ -504,6 +506,8 @@ def get_default_modes(): "no_half_vae": OptionInfo(False if not cmd_opts.use_openvino else True, "Full precision (--no-half-vae)"), "diffusers_vae_slicing": OptionInfo(True, "VAE slicing", gr.Checkbox, {"visible": native}), "diffusers_vae_tiling": OptionInfo(cmd_opts.lowvram or cmd_opts.medvram, "VAE tiling", gr.Checkbox, {"visible": native}), + "diffusers_vae_tile_size": OptionInfo(1024, "VAE tile size", gr.Slider, {"minimum": 256, "maximum": 4096, "step": 8 }), + "diffusers_vae_tile_overlap": OptionInfo(0.1, "VAE tile overlap", gr.Slider, {"minimum": 0, "maximum": 0.9, "step": 0.1 }), "sd_vae_sliced_encode": OptionInfo(False, "VAE sliced encode", gr.Checkbox, {"visible": not native}), "nan_skip": OptionInfo(False, "Skip Generation if NaN found in latents", gr.Checkbox), "rollback_vae": OptionInfo(False, "Attempt VAE roll back for NaN values"), @@ -924,8 +928,9 @@ def get_default_modes(): "lora_preferred_name": OptionInfo("filename", "LoRA preferred name", gr.Radio, {"choices": ["filename", "alias"], "visible": False}), "lora_add_hashes_to_infotext": OptionInfo(False, "LoRA add hash info to metadata"), "lora_fuse_diffusers": OptionInfo(True, "LoRA fuse directly to model"), - "lora_force_diffusers": OptionInfo(False if not cmd_opts.use_openvino else True, "LoRA force loading of all models using Diffusers"), - "lora_maybe_diffusers": OptionInfo(False, "LoRA force loading of specific models using Diffusers"), + "lora_legacy": OptionInfo(not native, "LoRA load using legacy method"), + "lora_force_diffusers": OptionInfo(False if not cmd_opts.use_openvino else True, "LoRA load using Diffusers method"), + "lora_maybe_diffusers": OptionInfo(False, "LoRA load using Diffusers method for selected models"), "lora_apply_tags": OptionInfo(0, "LoRA auto-apply tags", gr.Slider, {"minimum": -1, "maximum": 32, "step": 1}), "lora_in_memory_limit": OptionInfo(0, "LoRA memory cache", gr.Slider, {"minimum": 0, "maximum": 24, "step": 1}), "lora_quant": OptionInfo("NF4","LoRA precision when quantized", gr.Radio, {"choices": ["NF4", "FP4"]}), diff --git a/webui.py b/webui.py index 4eb6e89ce..85725efe4 100644 --- a/webui.py +++ b/webui.py @@ -100,7 +100,7 @@ def initialize(): modules.sd_models.setup_model() timer.startup.record("models") - if shared.native: + if not shared.opts.lora_legacy: import modules.lora.networks as lora_networks lora_networks.list_available_networks() timer.startup.record("lora") diff --git a/wiki b/wiki index 22951c981..b2b403682 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 22951c9818e44f3bfeecfd06b1c154230f711ea7 +Subproject commit b2b4036823c82e487fb5529c731a57b3fe77a1a3 From bef7789a2ca0b9326f8daeec27ee54375938c830 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Wed, 25 Dec 2024 23:53:53 +0300 Subject: [PATCH 151/249] Fix live previews with detailer --- modules/shared_state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/shared_state.py b/modules/shared_state.py index 3920962ed..69cfaab9c 100644 --- a/modules/shared_state.py +++ b/modules/shared_state.py @@ -161,7 +161,7 @@ def do_set_current_image(self): import modules.sd_samplers # pylint: disable=W0621 try: sample = self.current_latent - if self.job == "txt2img" and self.current_noise_pred is not None and self.current_sigma is not None and self.current_sigma_next is not None: + if self.job == "txt2img" and self.job_no == 0 and self.current_noise_pred is not None and self.current_sigma is not None and self.current_sigma_next is not None: original_sample = sample - (self.current_noise_pred * (self.current_sigma_next-self.current_sigma)) if self.prediction_type in {"epsilon", "flow_prediction"}: sample = original_sample - (self.current_noise_pred * self.current_sigma) From 9838f52786d6cc04e4c12c2f31b47b0629f5a840 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 25 Dec 2024 16:50:33 -0500 Subject: [PATCH 152/249] set detailer job Signed-off-by: Vladimir Mandic --- modules/postprocess/yolo.py | 3 +++ modules/shared_state.py | 4 ++-- scripts/xyz_grid_draw.py | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/modules/postprocess/yolo.py b/modules/postprocess/yolo.py index 8fc203280..caa711651 100644 --- a/modules/postprocess/yolo.py +++ b/modules/postprocess/yolo.py @@ -264,6 +264,7 @@ def restore(self, np_image, p: processing.StableDiffusionProcessing = None): mask_all = [] p.state = '' + prev_state = shared.state.job for item in items: if item.mask is None: continue @@ -271,6 +272,7 @@ def restore(self, np_image, p: processing.StableDiffusionProcessing = None): p.image_mask = [item.mask] # mask_all.append(item.mask) p.recursion = True + shared.state.job = 'Detailer' pp = processing.process_images_inner(p) del p.recursion p.overlay_images = None # skip applying overlay twice @@ -289,6 +291,7 @@ def restore(self, np_image, p: processing.StableDiffusionProcessing = None): p.image_mask = orig_p.get('image_mask', None) p.state = orig_p.get('state', None) p.ops = orig_p.get('ops', []) + shared.state.job = prev_state shared.opts.data['mask_apply_overlay'] = orig_apply_overlay np_image = np.array(image) diff --git a/modules/shared_state.py b/modules/shared_state.py index 69cfaab9c..2d7b54c52 100644 --- a/modules/shared_state.py +++ b/modules/shared_state.py @@ -161,12 +161,12 @@ def do_set_current_image(self): import modules.sd_samplers # pylint: disable=W0621 try: sample = self.current_latent - if self.job == "txt2img" and self.job_no == 0 and self.current_noise_pred is not None and self.current_sigma is not None and self.current_sigma_next is not None: + if self.job == "txt2img" and self.current_noise_pred is not None and self.current_sigma is not None and self.current_sigma_next is not None: original_sample = sample - (self.current_noise_pred * (self.current_sigma_next-self.current_sigma)) if self.prediction_type in {"epsilon", "flow_prediction"}: sample = original_sample - (self.current_noise_pred * self.current_sigma) elif self.prediction_type == "v_prediction": - sample = self.current_noise_pred * (-self.current_sigma / (self.current_sigma**2 + 1) ** 0.5) + (original_sample / (self.current_sigma**2 + 1)) + sample = self.current_noise_pred * (-self.current_sigma / (self.current_sigma**2 + 1) ** 0.5) + (original_sample / (self.current_sigma**2 + 1)) # pylint: disable=invalid-unary-operand-type image = modules.sd_samplers.samples_to_image_grid(sample) if opts.show_progress_grid else modules.sd_samplers.sample_to_image(sample) self.assign_current_image(image) self.current_image_sampling_step = self.sampling_step diff --git a/scripts/xyz_grid_draw.py b/scripts/xyz_grid_draw.py index 80336fa73..23e98e0fe 100644 --- a/scripts/xyz_grid_draw.py +++ b/scripts/xyz_grid_draw.py @@ -22,7 +22,7 @@ def process_cell(x, y, z, ix, iy, iz): def index(ix, iy, iz): return ix + iy * len(xs) + iz * len(xs) * len(ys) - shared.state.job = 'grid' + shared.state.job = 'Grid' p0 = time.time() processed: processing.Processed = cell(x, y, z, ix, iy, iz) p1 = time.time() From 33f2baa9a71a1680dd2aa007a306935a47b11df6 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 25 Dec 2024 17:37:38 -0500 Subject: [PATCH 153/249] size preview image Signed-off-by: Vladimir Mandic --- extensions-builtin/sdnext-modernui | 2 +- javascript/sdnext.css | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/extensions-builtin/sdnext-modernui b/extensions-builtin/sdnext-modernui index f6f629dfb..c3d8d9170 160000 --- a/extensions-builtin/sdnext-modernui +++ b/extensions-builtin/sdnext-modernui @@ -1 +1 @@ -Subproject commit f6f629dfbe63d8d6ed34f4ab2f9c7938914bf617 +Subproject commit c3d8d9170e16e029e31ef20d065fcfb77f9c64fc diff --git a/javascript/sdnext.css b/javascript/sdnext.css index 6e33551ef..26e8e2567 100644 --- a/javascript/sdnext.css +++ b/javascript/sdnext.css @@ -170,7 +170,7 @@ div#extras_scale_to_tab div.form { flex-direction: row; } .dark .progressDiv { background: #424c5b; } .progressDiv .progress { width: 0%; height: 20px; background: #0060df; color: white; font-weight: bold; line-height: 20px; padding: 0 8px 0 0; text-align: right; overflow: visible; white-space: nowrap; padding: 0 0.5em; } .livePreview { position: absolute; z-index: 50; width: -moz-available; width: -webkit-fill-available; height: 100%; background-color: var(--background-color); } -.livePreview img { object-fit: contain; width: 100%; justify-self: center; } +.livePreview img { object-fit: contain; width: 100%; justify-self: center; max-height: calc(100vh - 320px); } .popup-metadata { color: white; background: #0000; display: inline-block; white-space: pre-wrap; font-size: var(--text-xxs); } .generating { animation: unset !important; border: unset !important; } /* fullpage image viewer */ From c529508881b0090dd8339ab9d3c8edcca60230f1 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 25 Dec 2024 18:59:56 -0500 Subject: [PATCH 154/249] update vae tiling defaults and allow hypertile min size setting Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 3 ++- modules/sd_hijack_hypertile.py | 14 ++++++++------ modules/sd_models.py | 14 ++++++-------- modules/shared.py | 11 ++++++----- 4 files changed, 22 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 414b964d1..591c08d2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,9 +4,10 @@ ### Post release -- Add granular VAE tiling options in *settings -> variable auto encoder* - Add legacy option to use old LoRA loader in *settings -> networks* +- Add granular VAE tiling options in *settings -> variable auto encoder* - Add sigma calculation to VAE preview, thanks @Disty0 +- Fix live preview image sizes in modern and standard ui ## Update for 2024-12-24 diff --git a/modules/sd_hijack_hypertile.py b/modules/sd_hijack_hypertile.py index 69c4163dc..62f06cd62 100644 --- a/modules/sd_hijack_hypertile.py +++ b/modules/sd_hijack_hypertile.py @@ -108,7 +108,7 @@ def wrapper(*args, **kwargs): except Exception as e: if not error_reported: error_reported = True - log.error(f'Hypertile error: width={width} height={height} {e}') + log.error(f'Hypertile calculate: width={width} height={height} {e}') out = forward(x, *args[1:], **kwargs) return out if x.ndim == 4: # VAE @@ -155,7 +155,7 @@ def wrapper(*args, **kwargs): except Exception as e: if not error_reported: error_reported = True - log.error(f'Hypertile error: width={width} height={height} {e}') + log.error(f'Hypertile apply: cls={layer.__class__} width={width} height={height} {e}') out = forward(x, *args[1:], **kwargs) return out return wrapper @@ -195,9 +195,10 @@ def context_hypertile_vae(p): return nullcontext() else: tile_size = shared.opts.hypertile_vae_tile if shared.opts.hypertile_vae_tile > 0 else max(128, 64 * min(p.width // 128, p.height // 128)) - shared.log.info(f'Applying hypertile: vae={tile_size}') + min_tile_size = shared.opts.hypertile_unet_min_tile if shared.opts.hypertile_unet_min_tile > 0 else 128 + shared.log.info(f'Applying hypertile: vae={min_tile_size}/{tile_size}') p.extra_generation_params['Hypertile VAE'] = tile_size - return split_attention(vae, tile_size=tile_size, min_tile_size=128, swap_size=shared.opts.hypertile_vae_swap_size) + return split_attention(vae, tile_size=tile_size, min_tile_size=min_tile_size, swap_size=shared.opts.hypertile_vae_swap_size) def context_hypertile_unet(p): @@ -220,9 +221,10 @@ def context_hypertile_unet(p): return nullcontext() else: tile_size = shared.opts.hypertile_unet_tile if shared.opts.hypertile_unet_tile > 0 else max(128, 64 * min(p.width // 128, p.height // 128)) - shared.log.info(f'Applying hypertile: unet={tile_size}') + min_tile_size = shared.opts.hypertile_unet_min_tile if shared.opts.hypertile_unet_min_tile > 0 else 128 + shared.log.info(f'Applying hypertile: unet={min_tile_size}/{tile_size}') p.extra_generation_params['Hypertile UNet'] = tile_size - return split_attention(unet, tile_size=tile_size, min_tile_size=128, swap_size=shared.opts.hypertile_unet_swap_size, depth=shared.opts.hypertile_unet_depth) + return split_attention(unet, tile_size=tile_size, min_tile_size=min_tile_size, swap_size=shared.opts.hypertile_unet_swap_size, depth=shared.opts.hypertile_unet_depth) def hypertile_set(p, hr=False): diff --git a/modules/sd_models.py b/modules/sd_models.py index 184a242af..db541d48c 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -1350,23 +1350,21 @@ def set_attn(pipe, attention): def add_noise_pred_to_diffusers_callback(pipe): if pipe.__class__.__name__.startswith("StableDiffusion"): - pipe._callback_tensor_inputs.append("noise_pred") + pipe._callback_tensor_inputs.append("noise_pred") # pylint: disable=protected-access elif pipe.__class__.__name__.startswith("StableCascade"): - pipe.prior_pipe._callback_tensor_inputs.append("predicted_image_embedding") + pipe.prior_pipe._callback_tensor_inputs.append("predicted_image_embedding") # pylint: disable=protected-access elif hasattr(pipe, "scheduler") and "flow" in pipe.scheduler.__class__.__name__.lower(): - pipe._callback_tensor_inputs.append("noise_pred") + pipe._callback_tensor_inputs.append("noise_pred") # pylint: disable=protected-access elif hasattr(pipe, "default_scheduler") and "flow" in pipe.default_scheduler.__class__.__name__.lower(): - pipe._callback_tensor_inputs.append("noise_pred") + pipe._callback_tensor_inputs.append("noise_pred") # pylint: disable=protected-access return pipe def get_native(pipe: diffusers.DiffusionPipeline): if hasattr(pipe, "vae") and hasattr(pipe.vae.config, "sample_size"): - # Stable Diffusion - size = pipe.vae.config.sample_size + size = pipe.vae.config.sample_size # Stable Diffusion elif hasattr(pipe, "movq") and hasattr(pipe.movq.config, "sample_size"): - # Kandinsky - size = pipe.movq.config.sample_size + size = pipe.movq.config.sample_size # Kandinsky elif hasattr(pipe, "unet") and hasattr(pipe.unet.config, "sample_size"): size = pipe.unet.config.sample_size else: diff --git a/modules/shared.py b/modules/shared.py index 7bbb62e29..a4afc5b59 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -507,7 +507,7 @@ def get_default_modes(): "diffusers_vae_slicing": OptionInfo(True, "VAE slicing", gr.Checkbox, {"visible": native}), "diffusers_vae_tiling": OptionInfo(cmd_opts.lowvram or cmd_opts.medvram, "VAE tiling", gr.Checkbox, {"visible": native}), "diffusers_vae_tile_size": OptionInfo(1024, "VAE tile size", gr.Slider, {"minimum": 256, "maximum": 4096, "step": 8 }), - "diffusers_vae_tile_overlap": OptionInfo(0.1, "VAE tile overlap", gr.Slider, {"minimum": 0, "maximum": 0.9, "step": 0.1 }), + "diffusers_vae_tile_overlap": OptionInfo(0.25, "VAE tile overlap", gr.Slider, {"minimum": 0, "maximum": 0.95, "step": 0.05 }), "sd_vae_sliced_encode": OptionInfo(False, "VAE sliced encode", gr.Checkbox, {"visible": not native}), "nan_skip": OptionInfo(False, "Skip Generation if NaN found in latents", gr.Checkbox), "rollback_vae": OptionInfo(False, "Attempt VAE roll back for NaN values"), @@ -609,12 +609,12 @@ def get_default_modes(): options_templates.update(options_section(('advanced', "Pipeline Modifiers"), { "token_merging_sep": OptionInfo("

Token Merging

", "", gr.HTML), - "token_merging_method": OptionInfo("None", "Token merging method", gr.Radio, {"choices": ['None', 'ToMe', 'ToDo']}), + "token_merging_method": OptionInfo("None", "Token merging enabled", gr.Radio, {"choices": ['None', 'ToMe', 'ToDo']}), "tome_ratio": OptionInfo(0.0, "ToMe token merging ratio", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.05}), "todo_ratio": OptionInfo(0.0, "ToDo token merging ratio", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.05}), "freeu_sep": OptionInfo("

FreeU

", "", gr.HTML), - "freeu_enabled": OptionInfo(False, "FreeU"), + "freeu_enabled": OptionInfo(False, "FreeU enabled"), "freeu_b1": OptionInfo(1.2, "1st stage backbone", gr.Slider, {"minimum": 1.0, "maximum": 2.0, "step": 0.01}), "freeu_b2": OptionInfo(1.4, "2nd stage backbone", gr.Slider, {"minimum": 1.0, "maximum": 2.0, "step": 0.01}), "freeu_s1": OptionInfo(0.9, "1st stage skip", gr.Slider, {"minimum": 0.0, "maximum": 1.0, "step": 0.01}), @@ -624,9 +624,10 @@ def get_default_modes(): "pag_apply_layers": OptionInfo("m0", "PAG layer names"), "hypertile_sep": OptionInfo("

HyperTile

", "", gr.HTML), - "hypertile_hires_only": OptionInfo(False, "HiRes pass only"), "hypertile_unet_enabled": OptionInfo(False, "UNet Enabled"), - "hypertile_unet_tile": OptionInfo(0, "UNet tile size", gr.Slider, {"minimum": 0, "maximum": 1024, "step": 8}), + "hypertile_hires_only": OptionInfo(False, "HiRes pass only"), + "hypertile_unet_tile": OptionInfo(0, "UNet max tile size", gr.Slider, {"minimum": 0, "maximum": 1024, "step": 8}), + "hypertile_unet_min_tile": OptionInfo(0, "UNet min tile size", gr.Slider, {"minimum": 0, "maximum": 1024, "step": 8}), "hypertile_unet_swap_size": OptionInfo(1, "UNet swap size", gr.Slider, {"minimum": 1, "maximum": 10, "step": 1}), "hypertile_unet_depth": OptionInfo(0, "UNet depth", gr.Slider, {"minimum": 0, "maximum": 4, "step": 1}), "hypertile_vae_enabled": OptionInfo(False, "VAE Enabled", gr.Checkbox), From 26af293eaf590e4afd4a3f09c1681d54d67f5f7e Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 26 Dec 2024 10:06:30 -0500 Subject: [PATCH 155/249] - HunyuanVideo optimizations: full offload, quantization and tiling support Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 5 +- modules/cmd_args.py | 4 +- modules/processing_args.py | 4 +- modules/processing_diffusers.py | 1 - modules/sd_samplers.py | 2 +- modules/shared_state.py | 7 +- scripts/hunyuanvideo.py | 123 ++++++++++++++++++++++++-------- 7 files changed, 105 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 591c08d2f..f06d3fd44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,13 +1,14 @@ # Change Log for SD.Next -## Update for 2024-12-25 +## Update for 2024-12-26 ### Post release - Add legacy option to use old LoRA loader in *settings -> networks* - Add granular VAE tiling options in *settings -> variable auto encoder* - Add sigma calculation to VAE preview, thanks @Disty0 -- Fix live preview image sizes in modern and standard ui +- Fix live preview image sizes in modern and standard UI +- HunyuanVideo optimizations: full offload, quantization and tiling support ## Update for 2024-12-24 diff --git a/modules/cmd_args.py b/modules/cmd_args.py index 752ad02c0..a1e3473ba 100644 --- a/modules/cmd_args.py +++ b/modules/cmd_args.py @@ -74,7 +74,7 @@ def compatibility_args(): group_compat.add_argument("--disable-safe-unpickle", action='store_true', help=argparse.SUPPRESS, default=True) group_compat.add_argument("--lowram", action='store_true', help=argparse.SUPPRESS) group_compat.add_argument("--disable-extension-access", default=False, action='store_true', help=argparse.SUPPRESS) - group_compat.add_argument("--api", help=argparse.SUPPRESS, default=True) + group_compat.add_argument("--api", action='store_true', help=argparse.SUPPRESS, default=True) group_compat.add_argument("--api-auth", type=str, help=argparse.SUPPRESS, default=None) @@ -94,7 +94,7 @@ def settings_args(opts, args): group_compat.add_argument("--lowram", action='store_true', help=argparse.SUPPRESS) group_compat.add_argument("--disable-extension-access", default=False, action='store_true', help=argparse.SUPPRESS) group_compat.add_argument("--allowed-paths", nargs='+', default=[], type=str, required=False, help="add additional paths to paths allowed for web access") - group_compat.add_argument("--api", help=argparse.SUPPRESS, default=True) + group_compat.add_argument("--api", action='store_true', help=argparse.SUPPRESS, default=True) group_compat.add_argument("--api-auth", type=str, help=argparse.SUPPRESS, default=None) # removed args that have been moved to opts are added here as hidden with default values as defined in opts group_compat.add_argument("--ckpt-dir", type=str, help=argparse.SUPPRESS, default=opts.ckpt_dir) diff --git a/modules/processing_args.py b/modules/processing_args.py index 5ca6cfcd3..e0356dcff 100644 --- a/modules/processing_args.py +++ b/modules/processing_args.py @@ -314,9 +314,9 @@ def set_pipeline_args(p, model, prompts: list, negative_prompts: list, prompts_2 clean.pop('callback_steps', None) clean.pop('callback_on_step_end', None) clean.pop('callback_on_step_end_tensor_inputs', None) - if 'prompt' in clean: + if 'prompt' in clean and clean['prompt'] is not None: clean['prompt'] = len(clean['prompt']) - if 'negative_prompt' in clean: + if 'negative_prompt' in clean and clean['negative_prompt'] is not None: clean['negative_prompt'] = len(clean['negative_prompt']) clean.pop('generator', None) clean['parser'] = parser diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 33d875e95..1c2128958 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -479,7 +479,6 @@ def process_diffusers(p: processing.StableDiffusionProcessing): timer.process.record('decode') shared.sd_model = orig_pipeline - # shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) if p.state == '': global last_p # pylint: disable=global-statement diff --git a/modules/sd_samplers.py b/modules/sd_samplers.py index 4d1666ff0..5159785c4 100644 --- a/modules/sd_samplers.py +++ b/modules/sd_samplers.py @@ -80,7 +80,7 @@ def create_sampler(name, model): shared.log.debug(f'Sampler: sampler="{name}" config={config.options}') return sampler elif shared.native: - FlowModels = ['Flux', 'StableDiffusion3', 'Lumina', 'AuraFlow', 'Sana'] + FlowModels = ['Flux', 'StableDiffusion3', 'Lumina', 'AuraFlow', 'Sana', 'HunyuanVideoPipeline'] if 'KDiffusion' in model.__class__.__name__: return None if not any(x in model.__class__.__name__ for x in FlowModels) and 'FlowMatch' in name: diff --git a/modules/shared_state.py b/modules/shared_state.py index 2d7b54c52..bab30e5e3 100644 --- a/modules/shared_state.py +++ b/modules/shared_state.py @@ -26,6 +26,7 @@ class State: textinfo = None prediction_type = "epsilon" api = False + disable_preview = False time_start = None need_restart = False server_start = time.time() @@ -151,7 +152,7 @@ def set_current_image(self): from modules.shared import opts, cmd_opts if cmd_opts.lowvram or self.api or not opts.live_previews_enable or opts.show_progress_every_n_steps <= 0: return - if abs(self.sampling_step - self.current_image_sampling_step) >= opts.show_progress_every_n_steps: + if not self.disable_preview and (abs(self.sampling_step - self.current_image_sampling_step) >= opts.show_progress_every_n_steps): self.do_set_current_image() def do_set_current_image(self): @@ -161,6 +162,9 @@ def do_set_current_image(self): import modules.sd_samplers # pylint: disable=W0621 try: sample = self.current_latent + self.current_image_sampling_step = self.sampling_step + if self.disable_preview: + return if self.job == "txt2img" and self.current_noise_pred is not None and self.current_sigma is not None and self.current_sigma_next is not None: original_sample = sample - (self.current_noise_pred * (self.current_sigma_next-self.current_sigma)) if self.prediction_type in {"epsilon", "flow_prediction"}: @@ -169,7 +173,6 @@ def do_set_current_image(self): sample = self.current_noise_pred * (-self.current_sigma / (self.current_sigma**2 + 1) ** 0.5) + (original_sample / (self.current_sigma**2 + 1)) # pylint: disable=invalid-unary-operand-type image = modules.sd_samplers.samples_to_image_grid(sample) if opts.show_progress_grid else modules.sd_samplers.sample_to_image(sample) self.assign_current_image(image) - self.current_image_sampling_step = self.sampling_step except Exception: # log.error(f'Error setting current image: step={self.sampling_step} {e}') pass diff --git a/scripts/hunyuanvideo.py b/scripts/hunyuanvideo.py index b94c8b8f8..f61c00fdb 100644 --- a/scripts/hunyuanvideo.py +++ b/scripts/hunyuanvideo.py @@ -1,26 +1,55 @@ import time import torch import gradio as gr +import transformers import diffusers -from modules import scripts, processing, shared, images, devices, sd_models, sd_checkpoint, model_quant +from modules import scripts, processing, shared, images, devices, sd_models, sd_checkpoint, model_quant, timer repo_id = 'tencent/HunyuanVideo' +default_template = """Describe the video by detailing the following aspects: +1. The main content and theme of the video. +2. The color, shape, size, texture, quantity, text, and spatial relationships of the objects. +3. Actions, events, behaviors temporal relationships, physical movement changes of the objects. +4. Background environment, light, style and atmosphere. +5. Camera angles, movements, and transitions used in the video. +6. Thematic and aesthetic concepts associated with the scene, i.e. realistic, futuristic, fairy tale, etc. """ -prompt_template = { # default - "template": ( - "<|start_header_id|>system<|end_header_id|>\n\nDescribe the video by detailing the following aspects: " - "1. The main content and theme of the video." - "2. The color, shape, size, texture, quantity, text, and spatial relationships of the contents, including objects, people, and anything else." - "3. Actions, events, behaviors temporal relationships, physical movement changes of the contents." - "4. Background environment, light, style, atmosphere, and qualities." - "5. Camera angles, movements, and transitions used in the video." - "6. Thematic and aesthetic concepts associated with the scene, i.e. realistic, futuristic, fairy tale, etc<|eot_id|>" - "<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|>" - ), - "crop_start": 95, -} -""" + +def get_template(template: str = None): + # diffusers.pipelines.hunyuan_video.pipeline_hunyuan_video.DEFAULT_PROMPT_TEMPLATE + base_template_pre = "<|start_header_id|>system<|end_header_id|>\n\n" + base_template_post = "<|eot_id|>\n" + base_template_end = "<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|>" + if template is None or len(template) == 0: + template = default_template + template_lines = '\n'.join([line for line in template.split('\n') if len(line) > 0]) + prompt_template = { + "crop_start": 95, + "template": base_template_pre + template_lines + base_template_post + base_template_end + } + return prompt_template + + +def hijack_decode(*args, **kwargs): + t0 = time.time() + vae: diffusers.AutoencoderKLHunyuanVideo = shared.sd_model.vae + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model, exclude=['vae']) + res = shared.sd_model.vae.orig_decode(*args, **kwargs) + t1 = time.time() + timer.process.add('vae', t1-t0) + shared.log.debug(f'Video: decoder={vae.__class__.__name__} tile={vae.tile_sample_min_width}:{vae.tile_sample_min_height}:{vae.tile_sample_min_num_frames} stride={vae.tile_sample_stride_width}:{vae.tile_sample_stride_height}:{vae.tile_sample_stride_num_frames} time={t1-t0:.2f}') + return res + + +def hijack_encode_prompt(*args, **kwargs): + t0 = time.time() + res = shared.sd_model.vae.orig_encode_prompt(*args, **kwargs) + t1 = time.time() + timer.process.add('te', t1-t0) + shared.log.debug(f'Video: encode cls={shared.sd_model.text_encoder.__class__.__name__} time={t1-t0:.2f}') + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) + return res class Script(scripts.Script): @@ -44,6 +73,11 @@ def video_type_change(video_type): gr.HTML('  Hunyuan Video
') with gr.Row(): num_frames = gr.Slider(label='Frames', minimum=9, maximum=257, step=1, value=45) + tile_frames = gr.Slider(label='Tile frames', minimum=1, maximum=64, step=1, value=16) + with gr.Row(): + override_scheduler = gr.Checkbox(label='Override scheduler', value=True) + with gr.Row(): + template = gr.TextArea(label='Prompt processor', lines=3, value=default_template) with gr.Row(): video_type = gr.Dropdown(label='Video file', choices=['None', 'GIF', 'PNG', 'MP4'], value='None') duration = gr.Slider(label='Duration', minimum=0.25, maximum=10, step=0.25, value=2, visible=False) @@ -52,25 +86,20 @@ def video_type_change(video_type): mp4_pad = gr.Slider(label='Pad frames', minimum=0, maximum=24, step=1, value=1, visible=False) mp4_interpolate = gr.Slider(label='Interpolate frames', minimum=0, maximum=24, step=1, value=0, visible=False) video_type.change(fn=video_type_change, inputs=[video_type], outputs=[duration, gif_loop, mp4_pad, mp4_interpolate]) - return [num_frames, video_type, duration, gif_loop, mp4_pad, mp4_interpolate] + return [num_frames, tile_frames, override_scheduler, template, video_type, duration, gif_loop, mp4_pad, mp4_interpolate] - def run(self, p: processing.StableDiffusionProcessing, num_frames, video_type, duration, gif_loop, mp4_pad, mp4_interpolate): # pylint: disable=arguments-differ, unused-argument + def run(self, p: processing.StableDiffusionProcessing, num_frames, tile_frames, override_scheduler, template, video_type, duration, gif_loop, mp4_pad, mp4_interpolate): # pylint: disable=arguments-differ, unused-argument # set params num_frames = int(num_frames) - p.width = 32 * int(p.width // 32) - p.height = 32 * int(p.height // 32) - p.task_args['output_type'] = 'pil' - p.task_args['generator'] = torch.manual_seed(p.seed) - p.task_args['num_frames'] = num_frames - # p.task_args['prompt_template'] = prompt_template - p.sampler_name = 'Default' + p.width = 16 * int(p.width // 16) + p.height = 16 * int(p.height // 16) p.do_not_save_grid = True p.ops.append('video') # load model - cls = diffusers.HunyuanVideoPipeline - if shared.sd_model.__class__ != cls: + if shared.sd_model.__class__ != diffusers.HunyuanVideoPipeline: sd_models.unload_model_weights() + t0 = time.time() kwargs = {} kwargs = model_quant.create_bnb_config(kwargs) kwargs = model_quant.create_ao_config(kwargs) @@ -82,27 +111,59 @@ def run(self, p: processing.StableDiffusionProcessing, num_frames, video_type, d cache_dir = shared.opts.hfcache_dir, **kwargs ) - shared.sd_model = cls.from_pretrained( + shared.log.debug(f'Video: module={transformer.__class__.__name__}') + text_encoder = transformers.LlamaModel.from_pretrained( + repo_id, + subfolder="text_encoder", + revision="refs/pr/18", + cache_dir = shared.opts.hfcache_dir, + torch_dtype=devices.dtype, + **kwargs + ) + shared.log.debug(f'Video: module={text_encoder.__class__.__name__}') + shared.sd_model = diffusers.HunyuanVideoPipeline.from_pretrained( repo_id, transformer=transformer, + text_encoder=text_encoder, revision="refs/pr/18", cache_dir = shared.opts.hfcache_dir, torch_dtype=devices.dtype, **kwargs ) - shared.sd_model.scheduler._shift = 7.0 # pylint: disable=protected-access + t1 = time.time() + shared.log.debug(f'Video: load cls={shared.sd_model.__class__.__name__} repo="{repo_id}" dtype={devices.dtype} time={t1-t0:.2f}') sd_models.set_diffuser_options(shared.sd_model) shared.sd_model.sd_checkpoint_info = sd_checkpoint.CheckpointInfo(repo_id) shared.sd_model.sd_model_hash = None + shared.sd_model.vae.orig_decode = shared.sd_model.vae.decode + shared.sd_model.vae.orig_encode_prompt = shared.sd_model.encode_prompt + shared.sd_model.vae.decode = hijack_decode + shared.sd_model.encode_prompt = hijack_encode_prompt + shared.sd_model.vae.enable_slicing() + shared.sd_model.vae.enable_tiling() + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) - shared.sd_model.vae.enable_slicing() - shared.sd_model.vae.enable_tiling() devices.torch_gc(force=True) - shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} args={p.task_args}') + + if override_scheduler: + p.sampler_name = 'Default' + shared.sd_model.scheduler._shift = 7.0 # pylint: disable=protected-access + + # encode prompt + processing.fix_seed(p) + p.task_args['num_frames'] = num_frames + p.task_args['output_type'] = 'pil' + p.task_args['generator'] = torch.manual_seed(p.seed) + # p.task_args['prompt'] = None + # p.task_args['prompt_embeds'], p.task_args['pooled_prompt_embeds'], p.task_args['prompt_attention_mask'] = shared.sd_model.encode_prompt(prompt=p.prompt, prompt_template=get_template(template), device=devices.device) # run processing t0 = time.time() + shared.sd_model.vae.tile_sample_min_num_frames = tile_frames + shared.state.disable_preview = True + shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} width={p.width} height={p.height} frames={num_frames}') processed = processing.process_images(p) + shared.state.disable_preview = False t1 = time.time() if processed is not None and len(processed.images) > 0: shared.log.info(f'Video: frames={len(processed.images)} time={t1-t0:.2f}') From 1717327adc69c0ab428c315d1c5891f42b594aff Mon Sep 17 00:00:00 2001 From: Disty0 Date: Thu, 26 Dec 2024 19:26:33 +0300 Subject: [PATCH 156/249] Fix PuLID --- modules/sd_models.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/sd_models.py b/modules/sd_models.py index db541d48c..f44b8cb4c 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -1349,6 +1349,8 @@ def set_attn(pipe, attention): def add_noise_pred_to_diffusers_callback(pipe): + if not hasattr(pipe, "_callback_tensor_inputs"): + return pipe if pipe.__class__.__name__.startswith("StableDiffusion"): pipe._callback_tensor_inputs.append("noise_pred") # pylint: disable=protected-access elif pipe.__class__.__name__.startswith("StableCascade"): From af52b9331705d126d36f80d6262f5df769be4c76 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 26 Dec 2024 12:41:43 -0500 Subject: [PATCH 157/249] update wiki Signed-off-by: Vladimir Mandic --- wiki | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wiki b/wiki index b2b403682..04166d074 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit b2b4036823c82e487fb5529c731a57b3fe77a1a3 +Subproject commit 04166d0741dfb9b882884be2ac614b2b720b5e07 From 05bca5722da08a4365f07e7409bd9041623c84d8 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Fri, 27 Dec 2024 15:39:58 +0300 Subject: [PATCH 158/249] Fix live preview sigmas with img2img --- modules/processing_callbacks.py | 6 +++--- modules/shared_state.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/processing_callbacks.py b/modules/processing_callbacks.py index 39a6c66d8..048f53d52 100644 --- a/modules/processing_callbacks.py +++ b/modules/processing_callbacks.py @@ -109,9 +109,9 @@ def diffusers_callback(pipe, step: int = 0, timestep: int = 0, kwargs: dict = {} shared.state.current_noise_pred = kwargs.get("noise_pred", None) if shared.state.current_noise_pred is None: shared.state.current_noise_pred = kwargs.get("predicted_image_embedding", None) - if hasattr(pipe, "scheduler") and hasattr(pipe.scheduler, "sigmas"): - shared.state.current_sigma = pipe.scheduler.sigmas[step] - shared.state.current_sigma_next = pipe.scheduler.sigmas[step + 1] + if hasattr(pipe, "scheduler") and hasattr(pipe.scheduler, "sigmas") and hasattr(pipe.scheduler, "step_index"): + shared.state.current_sigma = pipe.scheduler.sigmas[pipe.scheduler.step_index - 1] + shared.state.current_sigma_next = pipe.scheduler.sigmas[pipe.scheduler.step_index] except Exception as e: shared.log.error(f'Callback: {e}') if shared.cmd_opts.profile and shared.profiler is not None: diff --git a/modules/shared_state.py b/modules/shared_state.py index bab30e5e3..b375a42e9 100644 --- a/modules/shared_state.py +++ b/modules/shared_state.py @@ -165,7 +165,7 @@ def do_set_current_image(self): self.current_image_sampling_step = self.sampling_step if self.disable_preview: return - if self.job == "txt2img" and self.current_noise_pred is not None and self.current_sigma is not None and self.current_sigma_next is not None: + if self.current_noise_pred is not None and self.current_sigma is not None and self.current_sigma_next is not None: original_sample = sample - (self.current_noise_pred * (self.current_sigma_next-self.current_sigma)) if self.prediction_type in {"epsilon", "flow_prediction"}: sample = original_sample - (self.current_noise_pred * self.current_sigma) From af3cd44d7f3a30af458d5defdabe77bf258f09ed Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 27 Dec 2024 08:55:14 -0500 Subject: [PATCH 159/249] hide disabled networks and add more previews Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 9 +++++---- html/previews.json | 8 +++++++- javascript/sdnext.css | 1 + .../shuttleai--shuttle-3-diffusion.jpg | Bin 0 -> 58599 bytes modules/lora/networks.py | 2 +- modules/shared.py | 2 +- modules/ui_extra_networks.py | 11 +++++++---- 7 files changed, 22 insertions(+), 11 deletions(-) create mode 100644 models/Reference/shuttleai--shuttle-3-diffusion.jpg diff --git a/CHANGELOG.md b/CHANGELOG.md index f06d3fd44..7c7f70393 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,14 +1,15 @@ # Change Log for SD.Next -## Update for 2024-12-26 +## Update for 2024-12-27 ### Post release - Add legacy option to use old LoRA loader in *settings -> networks* - Add granular VAE tiling options in *settings -> variable auto encoder* -- Add sigma calculation to VAE preview, thanks @Disty0 -- Fix live preview image sizes in modern and standard UI -- HunyuanVideo optimizations: full offload, quantization and tiling support +- Add sigma calculation to VAE preview, thanks @Disty0 +- Fix live preview image sizes in modern and standard UI +- HunyuanVideo optimizations: full offload, quantization and tiling support +- Do not show disabled networks ## Update for 2024-12-24 diff --git a/html/previews.json b/html/previews.json index 3e4bc3d41..a4ddfa7c9 100644 --- a/html/previews.json +++ b/html/previews.json @@ -2,10 +2,16 @@ "stabilityai--stable-diffusion-3-medium-diffusers": "models/Reference/stabilityai--stable-diffusion-3.jpg", "stabilityai--stable-diffusion-3.5-medium": "models/Reference/stabilityai--stable-diffusion-3_5.jpg", "stabilityai--stable-diffusion-3.5-large": "models/Reference/stabilityai--stable-diffusion-3_5.jpg", + "stabilityai--stable-diffusion-3.5-large-turbo": "models/Reference/stabilityai--stable-diffusion-3_5.jpg", "Disty0--FLUX.1-dev-qint8": "models/Reference/black-forest-labs--FLUX.1-dev.jpg", "Disty0--FLUX.1-dev-qint4": "models/Reference/black-forest-labs--FLUX.1-dev.jpg", "sayakpaul--flux.1-dev-nf4": "models/Reference/black-forest-labs--FLUX.1-dev.jpg", "THUDM--CogVideoX-2b": "models/Reference/THUDM--CogView3-Plus-3B.jpg", "THUDM--CogVideoX-5b": "models/Reference/THUDM--CogView3-Plus-3B.jpg", - "THUDM--CogVideoX-5b-I2V": "models/Reference/THUDM--CogView3-Plus-3B.jpg" + "THUDM--CogVideoX-5b-I2V": "models/Reference/THUDM--CogView3-Plus-3B.jpg", + "Efficient-Large-Model--Sana_1600M_1024px_BF16_diffusers": "models/Reference/Efficient-Large-Model--Sana_1600M_1024px_diffusers.jpg", + "Efficient-Large-Model--Sana_1600M_2Kpx_BF16_diffusers": "models/Reference/Efficient-Large-Model--Sana_1600M_1024px_diffusers.jpg", + "Efficient-Large-Model--Sana_600M_1024px_diffusers": "models/Reference/Efficient-Large-Model--Sana_1600M_1024px_diffusers.jpg", + "stabilityai--stable-video-diffusion-img2vid-xt-1-1": "models/Reference/stabilityai--stable-video-diffusion-img2vid-xt.jpg", + "shuttleai--shuttle-3-diffusion": "models/Reference/shuttleai--shuttle-3-diffusion.jpg" } diff --git a/javascript/sdnext.css b/javascript/sdnext.css index 26e8e2567..69d67baf8 100644 --- a/javascript/sdnext.css +++ b/javascript/sdnext.css @@ -220,6 +220,7 @@ table.settings-value-table td { padding: 0.4em; border: 1px solid #ccc; max-widt .extra-networks .second-line { display: flex; width: -moz-available; width: -webkit-fill-available; gap: 0.3em; box-shadow: var(--input-shadow); margin-bottom: 2px; } .extra-networks .search { flex: 1; height: 4em; } .extra-networks .description { flex: 3; } +.extra-networks .description textarea { font-size: 0.8rem; } .extra-networks .tab-nav>button { margin-right: 0; height: 24px; padding: 2px 4px 2px 4px; } .extra-networks .buttons { position: absolute; right: 0; margin: -4px; background: var(--background-color); } .extra-networks .buttons>button { margin-left: -0.2em; height: 1.4em; color: var(--primary-300) !important; font-size: 20px !important; } diff --git a/models/Reference/shuttleai--shuttle-3-diffusion.jpg b/models/Reference/shuttleai--shuttle-3-diffusion.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9f3aa8cdd3f385b34f2fae4aa33be1c50b775011 GIT binary patch literal 58599 zcmbSxWmFtN*XAI>B@on0#{}%pj0A4GDmB0WbBqV^@%K`Ye39ys*wYLEPR8;|N z00003fQm!`c=b|3dQnLv%KudcBe4Kb{_8&SOC>Mh^#GQ0vHF-b{DCa!l%a^i}@5|U#7=>!Q28ygQ7kLvYnDlrxY7P0?l z`PU5~z(9VPKME2d0GR*@g#hVaAAt5{_OD*n^U9#T(KJVAa3kUij?cpY%GUJ|Yfb%T&!XjZ5*kro+548V6_J0Q~^0;9tr_K3UJ9N#e)m)VvFlTnY^h0bHs5`;uKuwDT%BOsYzj3orr+-_MoQ{ z-nRlZZ~MpH)n(L%uuc!cm`~9}zOZ55ch#u|r?08!ptkR44}|UE9R?JQgzKB%8-0<~ z2nb|LB}>l&B2R|V^v4% zo35XR9hh?J-SS&ml(uGbt@79W?3~Gad)I-?t@PIeyskl7G@}MKgEZMFt-b#9gx?8p>bWpdoDs8{SgMf}CDNhihE$;;xHi z88RC@BQ8KEWrG(f`ZP#!85zb(Z!kPJIGQFoQD}%j- zrYTlDefCg?A+5x%0#gm`w5wY%cz@%Tn5x3Viy9Xshz7{tp$GL;7H6i8pX^u)oM;t0 zsBOPWYqLN{mtc#MJh2M9o-j8!q*l>PoQ&F>MgzsG=L*U`pDieQSCF7&7T_PTQR_L6 zw`!_;4&OD@F=^T9djpc-GobQ(em^c$6o6F%Wy!wrnVN!=R>*Ai-ahjvDts1}yS<`a zvuYW}xhaj}KC$vQD92D;&j0gmu>pvqzh2#6_NziiwyNrQaISeRsDzb09S}J%6DqSq zs17=elFv(O#A$GHqb;4FyB07q1G|eNY8N8fcr->X<0N9Q3+%Bfbv7g6dHn{$%rMkT z6}|*}#Qq)@ob8n(rGQ-5V7~E>X?<>HI@{|iR?h4-n<&Lw(cdHUXz3vVyh^PHQ^XCs zZj+&7DvY?a=K%y`_BPOM#zixubF;cQICVLhOER9jLLRjPWgy;?d+~Z_G$~1ux!t0o zOOqwqw+R6(q5GL2VpGPDtfGs*f%MPucY&_YPuDB-4l z3jiQPCcg8T_|-|Y#K>R)AV=-TO-5u#Y4WUc0^vl0$5EgiLw=vJymNUkOaTD>2Y4ux zP4La{ZC!IkH}g>2zs~>F+Kq4McOD#kC?B%zrq)k_9b;T!Wf`g!N36W5MCUP2tIWYC zsWed~VwtxWYz%(?=5mA9-<|DTV>e`fkBu(~i-VLv@g0*2x;PD*&_6&7i}6-g=yyBK ze$(bUKVK}5W7T&N0uV(XfPYkns}?{H^zm4C#B8t4;7f#9gxWxEeDs^O;$2MO`Gg2u z=JTF^`sCfevAZJ z7kVG?o2Sd&-oVK0ZDVt|UI!2}&bN%Xog1KOC01t{J*!*##&yE!0TRVFpr&?9zGJpw zwqMNFR~*&<$1N3K^{unJ>S;^C!7s+rRl0iuU7~!Z4X=mFspM1W-iZWj z{fd?IhrdT@LW$TR*lcAhy;V%@EjgsrYVw@$-^C-h8uvg6gZ9~Y5M)B#Vb0E7warC3 zI%!Qzh;0<3n3WHca;W=U%WkK};;FpqG+lvSK7lrcPYv5+@~=hl@5_GrG$v{P*NWq~ zj<3c~+_sob0&nKsS2O?GIzl#6U-1P5J^XnrmoVgJ^K4VY)GE1T{M_WOmedJ7Y_%}A zZRc=kHs2(j`h5ruUQhpti4*D`LWUBnyrco0>Q!qz7%C}0p^y{zRug57uR9TYZlW`% z@N?|00;)|mfQ})zO`wE2sysqnTh#Ep;SN^u3#itv4@E+Ak>m9TP7^(1-0LufBue%i z)9l%zyeh`^+y%AnLa=eVg=jC{KLAgixMfS1)oyi!I;(}{7Ys5ccNvLt>Mb1`dtD?X z8Rn~J-J~4ZZAvGbbrO42`vKpr>?*tJ%lG4>in`r@3(BOlyDLve$4mdl|J1@AF0UK- z`Wghk2$(U|U$cp~T!_BYlaTz;xTJ(BKcd%`@u_b-*UHT5*>f4%Sp%nc56WgI@tHBM ztD{hr0B$^Qdc;Nhmala2aK%^%OdM`?gQa0I>RKDMvkc`#SALgjhV|Q)%}R zo~mTXkNoAj*Eii0pOcPtR`CMIoK@{uC6q>;s;3Zr=Bjo{W>fI0`=0w3JW0fGo;te#uyF#ldW1x6?GZ+g$l@31Y zzt#Azy{O`^qfX**OVa3fawfOfUk!78ET@bSW7{s%7UqW&0sWu8t*wbFT%Hq{5!o^Z-IR-A@51PiQF&nGczjYcO@<9KJ;OE1w#Q|2hA$W!pru%CC4$U;d-LQ=RcZi3dqf+>;~qM@O2< z-`AU}mxEhvj8{2TM$eOAx=qNb1TQ5=xV3H!|OJHGcA?zoQH2CV0 zacBKy5cCfqQV=Im)xvu*j{g2adj!(}i!C08?+=3*rdu)UemgBnq@{;jC=lHny-7Dw zB;7so6{qs5daLg|q@ywFpBcpMP-D9i>%2$|Y+dLas9p}n=@oetrTROUP^%KtYfjCp z9!G2>megi*IKNa?soBpwx(&Eel6EQnTD~O~7!h_n(P z&74p?Y;dl`k}Xo^VPGELJRkyZc05W_Ka!@>xPz7c0i={2l-X4WJy&+C`+7B7%!sSB z1hg-4Tu`o04T=mDaS92M$f;!xHAlzVfB`9n7pP&Dk>o0ur(|orGt=c3#k=nLqs`@m z2}bCyPkQ7%6Y3*qd4!^Qy51W8VImWV7&7*}dIbihdU^UMe8bey1cF)-=PsN0g&fs+u zaOOjVxl8KV41o6k0p?UJi6VB%dd@)Qd-eR8E@%4F)Olh(bf6`-KmPzZ1C=rW7;%VH zCdr`dI`=FhY1HvYpeTi~MqoaZG4xGBP|^F>rC8l42Y5lrUW=-R3Q0wxS}w=~u5Qaj z1=tK{R6b*lmRTKVz@OCiCZF1jkw0tMD_z1^B{W81*i;_6d#TthE*OG|i8gh?hNr=n z6QII$AueCeu%X+VvbMQbRuDfG7P&9z@3pDSuC$yYCsW?x46i05fsSYAe7VFUeYTlP zrW;b0C>Ukj$;40wa3$Zb43>XqCYPF>r8v=Lza%;F%^ScVACC1;bFxRbb#Gv`ZH1E= zLd^g6tlT}ai`gwsAy_=_gKU@T;#@H;`~$SXs_k(lWcx6-?9$}I@9;3)4|^ymz++|I z6#kBQDAtgj{cPp8(Q)bXan@ttyS{3)o5!-0TLzB-VOtcyv~UrMus{~;nLuIddtO`f zMj9|{y6$?iO+4W+3T>LY@=1YI}O z0apCamHl|g3HEocvvei5)~s9>#s^)#zRVkewkS=-WUCc;Ht$ltTvM+%dJ%hlWyF2j zhPEMJE)QMVfr39*zm}N`YsjH3c^B#!4=kf;HDQ)ps%}o;3=fH z5zBh9OCxP=pv7wdv>q@Ov_JhdK7=Ng?*jRbhm4yqvc>SLOkwAv%N+Y`C4btU?7C7)gzGYV7O8e>8$T(2 zeRbkH;14NUNpR3oVTpMib|0@!<*y+J;VZBw$KXuXT))?us$`{%lw*G_#0Nu{I9$yy zz_U6VzL33p!@Yt8NdFrmuB{{xkR05`l`BBH{Q9I1o zB`p@Q8LBssG6?-Ga&^nmys7+Mn4PJ;$2yYXX`+An!871;yQE9{Bv$gk@-k5@Mf_~} zG`qN*f8G+*&lwG=<4Ez*$mT&QqsxvcayC8Wmn(oXwO($=U8>{K!Tmp#op_b=mngNd z7rT$;vXnHtfv~r*(g{gl*ixrUEVoOP_{n{ zY;?{pbgbC0e+2)2T5G;!N*^#3#OYtR1I)SGdw*2N*UImFZ|tg2rJqg91~C;8DcrVX z{iWe;?OM>*FV4wMP|9IR`F+FRN8nyFdC-Yag7tF2<-k5ERjx@nZ%|g`G0>hj9YQ=1 zeC5u==4;N?Z3!yZY{PS47@^V{U$>4#NNqyIzCAiR3?p=6fM1eREVG8)!pM&w{{gV3 zwClcJ2>YOSO;eaeXiW&!7l}h>xvY}YX#^>O(yac)3ofeR`nuSWtWb;ou zgrZ1Vb1ecY89E==?MX_ZaTy+$jPdm$)XCwyjI=cZPbj*Pxpqw%KJ74!)RXb};)8Hs zlU2XF3FNP9qmljnYaDx~OV&9f3Bq8PKD`gOSz_ABx_RlyaEHVzmPhHqlEDzcT?Dg~ zW1)0+VE$&@=@6U9?Oq4F7%pz(^4IpP^aINwUT6iah4h$Qk|n$TX*i9S`NU{EeEEQI zdGKoORK-)f3Hp8Yz;`dY*_^-6AgWl6_b#yf-l7SVSuR02IWkMuwJiB7_}L6U@BF=u z+}o|fYJc0BZT%65XFp@*I}!A?Olg35L;StYLIt!(;6`hd2q(dnrPlbvsvjfKM`71q z+udatSi(asUeArCl2qJpWM+X9cD%`g zOnEaTq@3*w^uQ-8CTUJMYS6=Rmi=CHyIJ*M_rvoC!z1YR>~1oplV{2@qk9-5gnR|+ z8e|%tFPd>AYG-eSPlrV9wk%FN2^nUsD0Wk7;xYi{)=z5PO5Bn1$D9VU_3?bCiXf+f zOD0F#ZeeaD>@l?_2yl?S27~cL&Zo1hmUb?ih)#drOb@KWzgG3xGaN&v{nC~))NbMV zFh|iDL!;>LKi1xzRkZ|gj~2RPxa5bSt!-uA1AjjU?di5n(~idznvJ*KEAyq&EK<#T z6*=-a@H92f5XXa5vaw$W`aZ<|f+@_AcPg+!{O=F%5C3GF2U$7CallZ&UUv0+u5nCK z(KfnV!FU*`UV8->6; z{@R7jp0Q&aHKh^`hSnvdxm0S)n%=ns>gwYa;jXSM{y`|Xt*f8@v~m) zgZ)&FQbJ{~!{mPam@}!zAG7XZ{WhYv6x1Sr*^8=yPm`%@4f#ssQQI)&4`0T5$YH7e8AQl(+jGIs%4subsTDB;ZUH$nY;hkU`^bJp_z(Wf3tn4Yweg& zd^QKY3o%bHkMrNfHAp$LbP?ufmz4Pju=6R&oh=FsYebbrLB{j9DEzj^#MzovP;-B1 zLNwq0K5K!8F3y+kQi9dZP+yPM-jVZW)O+&FG-&Bd9N*{SbRNpgOUJ?t{C+(V0^U;*b?VAdW+}V(yVrH7|9ho(A5lm``L!X5+IOn2@@=3@DySOD=C}wsu z={A4#$~HVx7;O1!+idv@|G0agV{T$EKXXJ-bGkl)Q1PE$i?y$Ay8$ zHeW*51m`Q@0(j+&{P5IJl6q?1c%JiwF|MzHca;usilj6yh27#; zk0irn@oY6xqET~QeWWZL&);nb2uvR!F5qgu7xK~W)o6JX6Tv6&b@>NS%qr}F`nlI# z?toWH7P8^QbB-Y4$g5w%x$8H9qsu9YOagfqvdX*bW zeY0+8V8`ZgXZh6teYT8MN3>OxSN$zfqddC3ySow;S9jz5Y9P_dtRD~0B-GA-?o9_sJg(+6bi>8|KHtSkoicR|mAF*d%Y z*nSMc;o7&Qu+feL0{;;7>znctVOBB=RxrV&cXhZ}q#_gAmNra3D04>8)~1z>6hj(I zalq}|P;orL-ocv0+PVAcSS!wi0GVz%u@z4(P}SD5K!%eL#I^ZXLsCwXDikI*cgoPi zbt4u4`ZRT$!uxz;ojSSoB`G<^v5M^g#qlGJs|pEcw}zw!=5{7xJ&JG|7xM^vR0>zM12X`vq4!cWg|VHOk?RxKeiILjv=@!bTR{o>Rc3j@wx!5PAH+G4 zxq$u@``pDXCr|7+6ffsQCZ>|4Lv=AEfLWZb*q6W`MhRAT@OCo(9o1D0%(K_YKa)n| z6Wyw;75r4%GwZxkUh86Ev`aI`tHn}J%5Q@$xxJ1(bF8%tZ}XudrbZInJ@FQb>;8%Z zlRj7R?xv3NbIH{;LB9JM_7CvM#_kU1b{l>ESgW6AVPKIbAXZy`Qh3rfd+#F;74I?Z zdLF0R$BfpYs^5GBdI80o{Z`o>Kh7GNP_imzXacSKq7#~JA2*~O3P&p#+PFG>A5D=! zoW4e`zfpEMgi?CM)~K~N{!VI5OKRf&P>pN+SAweXVI#X@jngV|)L^G=DG)?nd5X_F zc3T{uAkvW4Wh7DY<7;vyA|fW{YjqKNjd@G+mHw&9 z{E5T9O67YyOX^v+3jPeG=2f6=^~?iSxcnlU*^ZeRg490ij7`&Sak^#+``&Wml@V=( z)GR`4WY-~Taocrbi`NkB!Vfx{j8Qgn^0}SQ4lKiwW8Hwx^ILwNgxa3nJhE2{YOh?B zZ=@%0X(JlD6Go@Kk!@Y`P9i%m3{b2@`>G>BT}DHE8gogb55&xTtnHXwCtfE< z7DXIyotA;4^Qmm&$m6j*o!5YS?~x zs}nVwi2O67kQUWJo2NyBe}EBbp7oOeeE%;G`=E2B`l%uU!ZP!fDKn`laG)A{Gpvp$ zr~M=n!hk2!Q})W)f@`o(K9fZ*yW6L1HuFK#{E9bZO`_!U4iHgYiKZVl=$hpopdskN zQHr{;SI9a9r}HT?Ywe{B(_@ou_HGstIgavOSujpOmr?6&m0ebKw);%gipYK!fuZY8 z{=hp9c>eSkQl}=GrDLy~Qm3Gq=X)(*!W46K>ZuX3i)LiEVdZsrn7+sLNvv#f)G}ot zuzPz)b>kT*_<5Yqr*0LiDfcF3LM5h?wPv|d2}juHKfhF8^_tRZm0!=)v$?Zovo7=r zOJquH`sSG%8%Xn1SLx%P+tfNuH=_1!Iw^Jhj|RE&`D^k!DLkWvxh}b|-0UZL(~IXa z;=KyoLV2T;_#}7q3o5r80RDJ&@-F4zem|Jxr?y^?Fl+uFi+=#EY7!Dmox+M5c)EtHh}=u~lM-3|pTGj6S7Iez+}VG$)<(qRF(%ozdrKego6xCX46OI%HYEL{LgVXcvgFi@gzgVqzRRvEZ(j>+V+*Xflob8FyOGu@Bn5(a%gz z`yJq6>sDUETSop`rf{X3>R}J4QwNQUXrlY%nIs+e!Y9tnC;qcj)zzT(FF_}B6rWVA z>Z2&fC@3?&0SHEb8@5zbW*bkq9-6wW<63TAk5eO>U`@@gsvXzMCF8{vjB0aVD>|Wh zG7<4mh*ylumC5YV?ak~Uc#_z)ydW33CEEM0gL<9&`HT0rAJr)a}2Y`a6YV z$o;micp}8q{Lmi*5YA(B{VS&ueKj1_?&GHkj>S}nyBpBHVe++DjFEHBuRuvIr1sWw>L_0Vc~TSoGp{Gh z04nRO$JlY_??sIZqd;DKC4doz4YrGMB=jz;wZ2pDvA1&^zWc$3t>go0hAZn`U3D$l@1}|JTubM} z>gYA0nTc?%kLWa#RgyFrJdQ2?BzSlv?=6{Y1Q$4%yXw?jj0$FN}_gkAWC-* z)7ezlrP2*(zT$Hz5sJ}=?ci=kfT$f*znqzeAoUOvc!N{vWOc(?-Cz#gwe@v>$KufC zrt{FUBDutS@&Kn6zJFWcT@dsrd;J7m%a)H$d+Et`DGo}WQ%mw#t~{7`5)PG@p6_wY zAmeHXkfkaJB;a7TlvL)Qu~IvZ!Q5nRIHp`qq2po)^_O_$hv_#B3o z`IAiSu~i6<0rsDKJ*DxnXo5&UE%gJ5!9>eGJl3M$)#bVs_~y3_IH`rH0y>*rUD7Y zQa-3adeSda5BKLvjbVOol!y!I?)!HVc=*1U`ETU`KRHVDAmP&^?Nk56`oQ3SGze_v zO@Xl_w@?iz@5DO1ZS#xw2AAzp59>kAG#rxes3LcGUg=))VNP6^fPo`^@)`N)3guA*~dqOQJ7Imn%=iDwToJ(f!GB->JXm-0{pQ1T_7t`k5^(* zPnfsY!~?HYxxJjAMX?#CXCaNyQ6t8yA%WFK4}O@gHM94Ku<*=E{<|5)$$@eL@*&-B zILcM2CfUB-_*CKv>y;2;6)U@ticQg(r5c`5-tAu1;_6B2zoQ4Z^z`< zFwQt7s3O2OWbIy#j@s9IacE+bb3x(Nfp#_kVsIhLqfZL3tzs3e(4~AFiDBd3@PEi; zu{E|WancLg?m`@4oY)oCNVmyT){&!y)QwzMq&cUitjs>)k*+iAx$Csm^bdePlItKlV4t6S zi#}>~Etz|LsP`zfxhmI}C=k~^99JJnV~^+h5{M5r5sqQ{yXbgF9>6SoRsmE|{T&`} zUDR9ZoBp`OIl=R*1Qpi#TP@!H)X&#{sLa-FWRNDf zy_(Seqo=DR`&BE^W%GndFp!}C{@rbr5w@mBsSL>)Q?eed)+@WMgbrR9b$e#`hd6gW z_Qs3TtgP5?gwq2{?7rJATJP)tjEO^!yatmARWw#kzayJ$+lH48eT1#5x;h(pj)2f+ z$<`XZu7-JmDxq;&ETeAcPgT<*OC}3_NmI<=H;ZtS*n1lq`J+CEjfeUmP5DN2lALi8{F%y59Tk z{Y79l67tR_ZXlPi`;}SXrK+P`dV`}QRkAsjV}_^a51iTUd8ZEwvsv|8y9a9)4uHmP3(*W~^XyyiL#gc)EO0tL3jV z^tLmJiu829XcEF!&Y}B?Nr&ZzU!GJZ)#^cB)N?x>(#WzZ$SaANHy^rdU@Slsortyw}&qfJQ7Y+((!@sLzD;nOA`3}sDz1K-lUwz5V zVjU6>cibF8u(G>ftc=;HXT|;x&tpjpoz<^}WuTe^ZiX?+UNpLw^U#&{RFPZQSLJ7$>f?~yl(`;7$2c2X zw@JiXHM46Qi3VnJt+^puLh3~vhA$Ut#GRQMpdL{_yfJ#u$^~@_E(+yFdv(_T2V*&62@eXoF!g#MXu~Id~uC^6`jh#`}en zAS)d7Tk=i6jwcNIA_pHZadk7Q2(6GPaqH5jb&=W5rLL)N|ZC$KZ)GJsRdb&A)$a z!^7U5pVZFM*Ir${1&cUwDvqRsi`ympe-8{JT;3+VL(VS9S|8Mt*!l-}jq7l)MfiCH z(C4@9cDC@WI~GwOtc%D_J@D4;17~Jt%%xKR{xW~@>VSoR%Fp?OK|2;h%E9qWzTBHB zINS3RxBq8cdCt#BmBM$$dFq)F-Tqap)pdATE_Hr-?t3=YBWA1z9&lybFNMv`7q2@y zvPBreNyK)Xa59k|$Lyg>(_PgMG=UWLNzp&JFTf3G4FT5n{uf5zHPcz$!$#Xo+iJ(` z+3NnY3NKeRh(T8H4Ql4+WTLWx*LqQHx^D|l(ky#^+9mcugr-Iq-p!zS>d{f>W-EM7 z4(Lrqw3}8DpTvt5+I6!bCG=VTkT|NCbuXSage8-b2CS%)88bd}btqx{svp2S4WmWW zc}1!fRIaSC)shmDBj2Hjqo{@K6y}?2xqHl5HvZ14_y_nUR?Sqp?#AG(ORYZW_BiIA zF4Pf_XDo0#0gb7q_CdR4Xgi+H+$C4YTUDzh(u=R!q{UrZanvwY`N+s))aqleQLT7@ z7O+>b_GdzG)wL`qy6lIyFQfp%x_qUBeTlmZpu$(l_ z91k9b1XjPQR4NP@%C;vPVvO);@Os-^*pM*&uLsE2$L~(IA0?vxrphGGSG(A=vqkJxoqiJ^pDUe1=%(MxdIpj zl9Ea9wp&PV=ZPV^?AuOZE#uP77p0|g@J_u$L|XHth}n4)A_0W_XmhK*;xiK#88g#$ z@+g!_Z}=;sl2SP|r|nF+U?ZxcrbQMl2wM>?utGCFs-EyJD&_Yun(K1yBIYrsn!u=% zsLJczwrPjYoC-7RYEvh`8E31%&!)n{rQ1br!s(^oM3+k~B5+RcbRGf&D;N*JdWJtc z5nhfzBHyg!C7)`Ok4;nm=%iuC+lI8Z`l=B1aZ$@wufzpRLGlw71pIFH0y1h*uvE?W ztc*tc-%J^y(q0`J5I91)xws0OVs^5E8F?wCsnMH6y>yIDZ&wIZAgc_51<$Ht9mMLtF~ry2m2(S>Osj) zWfG(1X}PNJMNVb#wq!~wz|-okaR5o_K0^56JdvN|YZd6o>4OB)Q)!V)I zrOiSUzuiiV`6X!Vtr)qZ#Z&jpIQv6YQimDxK9b35XrU@QBis#=Fs!5xj!#%u-&ZP{ zb<0vYY*Sj?8L~jHC3ZXAiUc1beabAQ!XHRi$t_}w_13t&(i3k4B=j!$&%6YYWANbY ztwMNwC3H`011nHi5?UdTT)QXgr;c#n!HQSO`S*%TqR?s9w}wBUn4h|ilTXAT?`B*q z=hp>N!jy&msT`E`Ut(7B!QGQN05hwp1G9I42w<;1y3n1YJ-`D%mS0C^tEPN`~c=mU2*mM_|#$uy}br(&UV;3n$D|Y_sHVt z!#hLQl3l8RSw36)DkIRWN`-$FS@a)&tYI)6eic>$x~yj@bU=vvgJIwbC58rn=ul;0x@cKz zm-6;L*@68O{BQrZ_?1D6wYU9`K3oFMMw~=-`<*M~w(iwQq<)*_6Nf}1gMl=@!NQjS zlc1r&vY7HKl-%o5w~uA{Z3W1m&qFXK%%fbgg$=`LN~*jjA;HNC=WoJp%Y=R+n(Ig{ z^7HJ@FMW`Aqut_mr7ltog{_^=CGE9Q^G%A<#yDjxmiHMqb zy{Ns%%px`MdMuYUW3Mf$xEbN@&tGlLE!Z(DpNX?dtK#dcOq|5D5KXx1yRF}osJ_jb z2XVb*-q5DYbfW&~eoKD;uF+}aatI_?%E@vYPBWnHAreV0+rX6a35Cv6tu?qN9fD^dD)8W!o1(y^N0ilx#w*kkZ zKpHVeWplSx-C`WiI|=+yUS_% zaC3@sk5?y~0z;(?7$QjPN&CUyt|h+cB^55l=iZ-Df@n0Pns$-@RC>U#0^~XYtF0Sb z3hMFKuJ#^Y7aq@k)~QB510E_h(xeq=OfF2{9CFb875nuz)?06{=Yz37LS(R9+YTSx zEY#vX;)c^P1t+7Hh;j24?MzAeXy1ZyiN02{N6mk8Xvh{@>JG14YDB3OIAyq8)^5;? zvi2iZ_?|`nIlpG<&5et*#su91q)%2LL@^61^^iq`T?t`-BC?TxK6+wR)$F^ZB*Eja zo9g)Ptx)IC1oLP7LrC$u9BIT?lMMOOx8rMLpXdEQ>p^q?f$N`c1z`^{S?%=$)5`1T zR1l96(ew{;4uUKih{Y0WrW(?#s1uLQf%OTA9u)W z=^k;>K9C0_ymy*8W47xSm?66Unr#s(dMg>9c(N2xN+!XUSt z(VW#nb>cTD0{w|LRMrc4{DU7p`Okq>U6|1xQY3r%w|B?=br>Fw{YF6^Q@2-o3^O9$Sh-Kq{Mt{8pZEBSz)VcWDYeaSP0B@^G~xDcqNA4Tw$vtG^aGxuoJR?mJn9V) z&B+SSk^j97$rog|)*YJ;mLQy8HE@qo6Dq}Bi|o|}FQm3gyM4P1Hj(Qpr@v`e+`Ese z1Vk#qwNyT(Bs)AWWTJVw__7R9uNlJ?j`Ces9`7&iVlYR^wJS?BGO`=;q8AF)ND;Hz zr@^0Yd6kLcDTqB5qdmoF>fxWUPIG@%y%9>wV~eljP%ZqBq~V)4)wUDFb%#BZWz0Sw zyN5)9IGc0}P#Xv^$ayhWIDR*W)<~o98!77Q}q}QZT7uDpg1oz77 z+wtAysx4OEQaOm1KZHIhyK-gYuhAt=MFJn@tUFp0Hm%&qP-22zOMX{T3tN)S8zWja z>?CtaYFCu2{fRBuZZmI%zboz<`GxGpZ<9UZ1LnnxH@n7-~_HA;Ci8_k4_ zh~1mp;-m2u#H4Z-_R>R=z`IB-6}?B;zoLj>t3E{yAcFF-$G#FO{pEY4ML z$1Fb5RX!!JbW&4X=DV%Prk{1XX7@_jdh6ryHO80oK0Esw+_-ZB`({DZPW1g=aSRCf z@~J--pqF#J#*kyQ_U(s`TjsrkFRk{IV>Qt}=gX$TMqo(sP!r9;$<^GQzX4N{xt}L~ z^*tRezEIaVvmv44WP_1^%MKx)5aGTRr*yX;mVxj0Px=4Sbr=K}-V=~SYYCYyJ2rllASBu9 ze>#n$!5JsR6QwNrFF6BIyBb$ z#+eIB(%ju5;#CjGAmb!^^{y{T)L%^0VY@&`Lx)gtxld94eX5k&S{lK^T2x;%(8iSp zc(8UK?LI%PQC}QsW;Q7V>iPLrLU4WfCWTU?)B60)-7G9s?!NE%1-H`Mksg~0`WJF) zr~D;eb+|V6@EJde@<{xJ4SKyQr8NFEvKv z_rd_f*dEn(tA2++J{H%r{0X%;eK{3(TltTiEND0$rm;$7mhO7iiY;z)POM+Tt@9!U z(YK?pZ%}!ue7tqwRancIQFlH;ZLN;Qdea+&#sKb3OK7zg%du(}Dw(*qlsEk|+Clv5 zDnd!cOLr+GhhE%$re-RoBmM;^p#K0Gj3HxyGH4p|Fw6`Sdi=}Z9jOu7Gqv&UUS@}0 zqYX+HIUghCC#a@}%rOr&d1Fjrwu-eAH#9?UCL{o!4OmsSbs=1-_1i;;DeJKNH`^46 zk_sKbdy0x@0m8>7IQiHQ(AIB4m%LtUwE0>v=VDQM_7m??P*Ym6o zF++X!{{X&d_N5ETf39=|Pf?=T2<mvyd3BKPt=HzQ=0IpuW;N`7gtDjXx^qt>(D$fR_$J_2_>ZrB$;hgp3`HS?-a) z<&=MwWh}Du=BV~7Q|%ixR1`}3v@Lb9p$esQp4DhPJeNY;a5IcnQ(W7fRA%Yyb5q$| z^Ofed+$rbIH`6yi7h7`Q}wOs5r4WW9W-woQ|x8gKamc{1RRr6T!1FvV~V(5 zD#r0@qUL3xG+YwERSLd#9?cqY$r&KV;Gay^Z8cGe(oES5r>MndEySP&55TKc5udSU zrWfDxs!?9DADuxS)cMgq%1E|7h5@H7!bmcJAI7TW*`idILHt|3y=ki=e-e@Or*T-a zlQ;K6RAb4;e84`mTr69TbsRK!s%GXj7!nio6cn3AkDU%WR$^U^-~e&zX%wX+eh&~` zS|~`SeWRdkFdv;{tgts+=i0NCI&$3_M!k6aGO?=K-Q~-VGI<*fxP17*^%$(-<#u8} z3Tm7d$p^Kn-@Mt;!Q!jr!pgDA2XSIJ`u5lRClTh9GTN zv)>@sERec%&OIwXs?h9ivABo3I!U$5+2Seoh~iQ(Ok)K8HP3~c_;{*RDaTZhJVi-p z?r6l31Gq>52C1l=bH5dC;2xhBATdsxe%1YL@UPnLLbubR>1Id0^@GIw6au`XjY}5!^^VhQL0;Co`USXP(3+a*FL^8HEg?rNOv97xvovl&a^NWn3mV0|lP8*Fk)t^OiQ``FqAwTCVc{PQ2< zKBLf8ZzW-6X-df5e4u}sp+e5&N~@CP=vQLgvB}SK-lqNi%O8}GgP`Cjsxr0qEAoA7 zPAw_M8tPSqAK~iXmufUCl3k3#tbOZ3c_4(8fdJdpoaCCN71YhPv?(pSw`!5D8csU< zDB_Dn3i2D}rz3U<>w{63U^rd5qnQPnA$ZVX27T(eV^hvPQbsI)i;9dkK9sqxJ6KbZ zPR>OGBPW_tIjI<+@rq1hn1>DrwKj`@?ZkJa27oPEUOl9{)CL4$l=rH=DwLYl65^$! zI(RN)xQIxK2t9icee0jKxodet1M-veFQ~4F*Y0!5hw_~XxJ=x!;fk5Wx5l#(ZD zLhzt|6tR&is_J#;%Yfq`B_C6{{Xc4;N;ea_H2COM;RF76;@%$ z%5IS|4=;ZI0PcavtAk94XZMnG?kE?skw!E>c;pe&b!eR`7x|iLH+@ATMM7;{&1n~Q zSwZ&|q%??##^?Ph*_rz`8ddtkFpY`w--FLX+K=prSD!LW{uNP^j>c6mimKG<71L=4 z^EHf*BvFLi&yMVQu9aCHWU|V)(Amsn{{Z#t98E7=)Ap!`msK`{#s|#XeK0dv-)fY9 z2tQFz+M*s`RMav@o}m5}m@XxA$^rDH?GXN>Q5q&tI%2chR#DR*jW1|}>T@l<)<6AK zM9TQ+w7sEA2QI_pzL1~we0vJ4(to@8(&sf;X1Wr!%bu{+Q!o2exrhtRWW@h_ZKvC#M72V%vDde z$^FHno+9OC!NpYRqaT0k+-*a4V;6})=XEkNB3|&s+DM&x-#f% z2&*^yO5~lY7bNjd+Q44QW|)eyPNOm8s+0K9?P5P=Gh>D7zCgNXnWs9Var>&&`Ec5Ki`z&(RdZhjY8{^WPB6O8ke?@ zVY*cw@;Y@a*bK~Z(hrLzu0?vBnww54Cgu0BLd1 z53Oe^w>FG9ON}Iw92aM9<&9KZ$5FyAeNQzU$kS5ZUvHMU$MG80@Sc-4u-6w3+aQ2^ z?wq(MrUB#Enrh66O-FMy#@6upj}si83vep*_BPR7Gl!a79t?yZ=|Fo#%4zn+h}~sl zKfqf6n8ts|swD|wPRImUBTA!3f+G%{RB$>IJ(JvT z5p4`xokDAt;|RS1uW!fJy>{2&)`HNhwbr7{%l-A^{{S7VeWF+2LYs26+2i*1=2e*W&nFq|Vn?qVoDU-~(c#AkQ8CUTWhjtCP(X&m_1e1qiGLdbb4s0QIY` zm%!S1m6z=oM`$;{{X(1{3!M{1@)>&5JI+l zS8@LU2qv0;<>Kr9`d{#)*iW#1YL|0_*xy?3hk*3&`T4s40KS*}DScjJ`rR{t$sX0{ zAMk-L+!`{{qepRk!fTBdpT73g2^uk_FP z_`3f9zL)$c_7e}SQ#|`0m3mH}@Y77ZyGTvd;8WCwT>k**H8Q0rtFuaVqf$yS-1Bag z>h18WL<~pUFStM7PyQyKvXSY1O0nf3je2zd01Nauqi0Q0>6kV=ysm%r8qO)Xa;?rP zzq-FSL(OBAqixD?YkvO#!*zT;Y4Z^9>|-6rnq9Q{j9msW zChC6DDf1#784Khp;IaIu2+yBt9Fp82gsoB9F}RG%pn=XQ9wr0spURg!O&(ubD()qD z8JPP}*;=diFm#^N8f80h-2G}c2A$4#L~A+e+L}S9aeFb0Xd;V?lwmZa3I$>^X~YUb zu|{>w*v>L5Zo^M)Lu3y$VkT?^JT`rEPnP65iZ0B_o*PxRiQ$~Ir-zJ=p82k>_rZ`o zq5Bj0S8;-TsT)3LrVmbiXlR(!P402$7{?hM$3sfU$i*E>vXTfri0e~hM#V)0bU7I( z=}sNGepJc0T|<~GnQa9g=kJ2wrE*y>F_Mn|@fQ?g5D(|jwZqzU$UriDDw6K6gD z0A94d2^ySku+O$!2IOjB_dx@xNap@X$wajrO*EMmb+rk%D9!=)H8!0fdsK+<&mc9VZDTr(-&0N)#+j@s@P-7kIKf~6qrD*o{w+^i$FC}_cpCzt`bS2z6!(Qz1MD%-CO-!JePCy9R5PQ z!(7*|d_k!_-RtCT`6B)lUc=vxr|Vd`&ResxG^oR6e^aW|bzMsONbcZ;^oGe`Oe}W zr*L^6L-<#mH{QzdXh|S*Z(`-frE_sPm8Ot3c*C9trBV&NWK+2s^*TK##n)DBBfJLY zAMbw}{{W41z#QQ4YLuGThX}pV>b4iwx|9NISyj~b?mnWt*27V`((Xj|?nn2RvkK8x zeM}{Wi}+7u(AOH8f4XX~gf&QYc82C7hQSV<)X`c>RP+oiB?{>?Qf&FhJ)eqn%6!(x z_v={feXMpzKdE??mFKc+WZ|$J4r?b?jlZ>KI5^+wSg6HInQnZ}H8)mK=v(mjh9K82 z9t(+{J&WdLF_KBF+uQ5iJ>! zNws8idYp{%1;Ge9d)0R=_{k(Fz|CnXT&i&8Bvn#5P%6qQ6Uqbl*AFv2=u?CHL@E(_ zHCN7AngNu+P}dr27p z@lfin=S(m*TNQrH+c%66?0FSs;gSam#X&xYG`*$vXH+myYht&T-mNN0C+^jE73>Ez zdrKO;40kH|WOl1wOAe|>;YT@Z1p=W;vdN6+u+23i8kl-8S8S?NH20ZSE#PG51Ow8o zT=S|+4580`0yxwHTY@@Nkg~80%b#kQHfuwbN172&)M?q9?Tl*l_SDQnQv+b1q* zQ%#lHCI|0R?hes~JesIV<~YnWm%NuRmF`9J0f0?HcYD^lGsykzJucGq22>o7GJR?{ zB-JF0rBZR({KRslae?>>h~!h{Mo$x3)_Rdhl&zYxnoYih45U-o8fH!`frS&Dk(Te; zgE=d1Mrp(6P}Iq_%erfX0k`eJJ*jM^wlOR-Ba*C6O0XXD66tka&|{i z;r{>!M{{o--laLVf=qd6pl@;O&j5cq)P^gKRL3Wpg(8oRa0hZewdPL~HutgVLkk7I zrSst556J6o4AILV`3lS#kGUAdG-+2Fy0z7jNal@HMBDk?4}VVef~{8i&r?}ZrEi4f zyiMUrv{{HQvqFCCi`&_UWBlu|I&I|oe74$$l`~0h!R0Ia#C6LmpHL4V*HtW2zby4R zYGIcx;&~0kJ4vgT$}&%_dM62sBH+}Lj!gw7Reo{67#QN5*v?0-B8Uw+v7QDg42+6W zGH^MdVgNdD52^1*UqMJ?p^iz3iYUCtZU6%(uWHrs-lqnWcK369#w0BY705DnC)5v0 zsHFEZsX0Y^s^)7!du?ptvFCzMuO5}!c+O*IXJGcx7~*6H>5eh} zdec@!b2_@&8S?3O(j?PaUHMZ&6;??I*i`k%>)N@^LtUD55kyRJ@JI)##Wy9;P^CY# zcN(6d4VIN?_O}SmV+Fjkl`W1 zv1Z`ipZezC%9&!NZVP{~-^zu#L?J<`Ukx8lYMfC@Oy1LDQ(*}h+$0#IOo*&j(r)0s z2(9DroYPmH=T;+@HBlhjJUDbdqPiVg+8ezRJCtP-!5G62;uJ=u{nl%Uz{;P$4U9MO zuCPc&G5Qadvh?WCHyGYd zK=q{)H$X9(15s8sVd!X(OBzPaoI37bahge?xU@NT#{U3&JHMS*VsW1RMLUJAStpG5 z6$-g6^%U+a60?oD{VJ?amSjJ}(w8v#Y;_(B*JROkxa807+~Jq89sdAN*`DPtwG%tEahH!gFe94cbjydlv__1Rfr{yjm5Zs2#l~LiR`faxyPMgr zcO+@BoTwmXs5O?NfVS4cjFX<#Kf7$N-Eq6yW=l?~r_~NC&Z|sNMH@{A(v; zp+zmr0xLEu&AbqK+Jl-%quhHBsjQl}Q9U}AeDv7K5NLrtfHWc?A2R{?)_k!!+^MZ@ zK=Fd72pPK*k^O2JuD;K3yHA{?a2LI87|Pd2Jh6~Zm6R_b8^aaY6({B=MyOxp^ zF_q&joc>;w)lWsMvE||_xz9-4sM4&GG;yks&A{|O;Za%u(w~?s0!ItT82(o$M_>PirW=dCg+)U<)$C0K#) zz@esg(!}A~aavorQG@WIiU>G1C4_Y~N1&(s5%LzM!sy9(lYAj*52O&veRS0uC zBMfdfRG_6(Uy>NnDfOvmDwfSKJZ;D#sb2OuYN*Daw05yuYIDFiIH;_hOa!xJ;IaO- ztfOXdP^_v&$!I$&N!p`>f+`CsLzf2}lG)EDhZ_@16DqH9v%0K^xckTQs@L-EyA6!0 z)y6s;n2baAmblx~wMOCqJqL48Urr@;WdMA=y3}+W)g7sQJ2;InGn$Nb7sN4^FXu@jer9QW$gpW;{-Qg|KGg-4oOeDSvRg2bskq!khmzY) z27NxX{wJx6Ue8#bapQd{G@lhu1bFh~jV4g3umHoL9-LR8YW^Fy@ipu>7eph&2b4{` ziRKcd@N?4{G_7sUg;6xDc|0eY>wJHq#iVL>vc)Sm+Ttu@Wy2=Y7oV^6p;?XXb3aDA zHkWYDpc4~@1J?(oX2CAfDKwh15^X!9)UEW{mIHfXa9+Yz4H}><%e#UI9SHuF$Y|aq zywdabd9aW7<8D>I5Ad!Em|6Xoe^b7mEr8y(J*XNWYDV>>P$NdF z`_Qo~@|^wCp1+ka^VWfk;GTFDEH?99OtC{7Nh5F-dYV=p(%g&G0oJ+;FAHjEs6>-m zBNE#}H_i(YkPhCTUYu1ZGdET#Ugj;VO7~DCi@rCN1Fmtn78w=lc9Chm6@t=Jr`xsU zk!Iwp?-94C?eA5}k&~|aBL4u0uWqz*mzsJ*3P=006bv>;83c93a(AzDs9Z|dcLfuc z4F3Rh{e3CB?2MrlwUZ#$d_i%mpc5XP?HmApl$Q3^kZroWo)Kxt5^n`db`-5AxoN^0 z#cE(W9o;KeRhLe;V8&&`AG(~C{3$rt))hInMFDLbiTkoi#!YmZO`KBreBWDJd8SEr zF_~~w4uB6rJJQe_X`HBmBFoI%6ngiqE4?Ld(sU8d@(xi$Wn&({{<#iv#~Ay{2{pQQM>N&sc9y!Tt(4XYAe9(??`g?X>0C~aERav; zCUgAite&N*RE#+8rg=?}{#|scF`!_aceFANR=ql~3X%o^GM1&z@QmNc7GPalH>tXD<{i zO1liBK0hpGwQQh<<4kpuWs76wNjT@WPo-q#X45J#yi}1!=Eh5Fxgjb^MkDa*59M7n zb{1NbLoJl)9I5y5^JnYLB{g7l-R|}|Zzmk}uIv60s}^m{AN|m9D!GWV?qEZwPk*6B z9CnAyg_xtWfsjr}#cJwsYBupadQt_B6dBw-PeJ_hX-Aq;v{t5)P>gw`^f@`S`QwO4 z(d}0K4|>*@Pw@m32AQNqDp^)#2h1Y_x2ZfDx|HfuT+&(*N>pmTSW4{YK-;%Hs(a;R zzM3{wR#xOEKz*o-y16n|lSK{tZinGr9C}r~?t}_O63mJu4hPO#-1;7vtofHUja4_v za|ymy2el)7%HK@ZrU>eMDR%L+w&A%WaQ?s2sAvu4YPRmj2+RY04+r_yiP*kgXxQ$4 z*LAe7m~eXYQ|fRFSrIMQamG&JRGb>Q%;7ZoQID-$$0o~eq+y);R(0ADbvE&aJw;lT z_A~Z;y1mHVV~mG_Mphn&r8>$u+WruE1^E>c(Olk*KWkAzJ6xwN%+f0?w^BAw3VYXA z;~h5M*22}{i4e(*l{w923W`qFJMkC?Mh<+}wuc#TjpULC9<`;YLK4kXV}+}e(DUJ| zC`Bj`JM|#s3Z)yetFK+%YbOm(xMHmqx|?w>E{{WAcT=TSXRjuyF+L}ig1QXZtsg*a%bkN39b=}^U{Ea0_p&i4UeYrcl&Dp4Q zrF$LJ^GXUF*6VA2rjn^RBc()_A~++!$paXyT{&LItxqwaprt2kmm^bA{iF@$MnF(U z*A;_=j;D;I?`N@oCCI@wlXHKv2*+K(e8BXnR3|2ISIjBGH+@YREtVMMos6fkX2)~s z-l;yQvZ)2hJO#ny>xvaRolZqkjGm0$(lmJDjczY_g^uR=fGWKSVb{1nj%r9Z(ERqhcIj`{UGqg=C#czj;{XQLQ%gvKy>;deB2P-kQz?E5B<#Rg@m8M}KO_ z(|kqtZ8qai9%4I*3aXVW&jYCYy!!Kj-mj9RUy*QART56<=caUu?qUfn;1prT4{`b8 zs@}*F?50lkA&Da#V;`UR*P#eqBaanZ+0IFIEy-}u$FWo8JQpV=xT+6zA;RF0qZOoK zZO$r~mnCSs3nZ>abA|_l(~nxMy6${>FQ`|mjT=&fItJ1y!!R5>E=AC$@=!Irv(&7Hyb+g_rBsu?u*$)7I{*0 z?kbBuYSdD9IVFgxI%=DGA$-kEYX-(>+Efx?4B(t*sWUl7=S`-q)~Bc>!yuOBkaPIf#*l|i zhG7h#HNq9QlV(9ahtrCsGa1u%(Ae>2g=gYDHrU@?eUkbJnPMt;B9pZI%ikah<}YC{~-`<54xV@!Bl3$5|uXsZDLbOZexQg=4hz4kssYW^Bmz)lq=c%r{ z#JWbmb>aIxH%+l=#nH{%DFYiyl;n^A$^9udlibpql$NOTRhWv7SSGi)dwFAImNqEp z!;rw>ie(f1p!(I<@W#0olP;+?n>03&efN|EBb8}O~>y>>T-wX#qomgYq& zyD~Bqk%CA(S9hsv_A}dC#R+Jlg#rsiUyuRYIp&>-bzek7YxWH~OSp84LlwM8Hk6c& z1zzUXh51?VVF#JKbBQPwQAtBYyxUgf8?^R|Q!mOh7I9Xoy%CB4+P z(+5@{j+moIxuU#wNatjp?ikIh$s#6lV4dhZk9F~A81zIbK8T*7?P6hge?-03^#z8`-DOl@ zWZZC|d)IXgx>tw%Jl2-C5}TPqGf0FCk)QsxK3?QXvwB3=*6;2#-7Ljr(+^r*pnRb=vD%HgBPv$UTlZ<3mlJylg zZPkhART^|GF}rUx+laioINLlj@+tHe+7n1WIovW0SkmpCRT~^Q$flY$vu56(adi#i zJQ8hq&Que{Rt(5@FJo4%)f~-To{6ST*M1zmndAmW+*>0Y)gKK^vv_Y&B!eM5udfw_ zNToGltEAD)9v2*Z+;dV)j9>2HgPd1O)QG!X9IuSu&6eMvnmk5-6kH$78m;1jCDi02 zf&fB3gCPE=@~&B}PS`f$n<(fpM|E(P_XjGE<_P})zm5p}Ju0jcyfQ~|9}gRD{73lK zPL!wtCZY72P$ySNJ}T(~16tzAg_tgIyMUOuN! z<1xENvpA1}Fjwo|r16w8uZL~qb&M`X-MxPr<)yjrFguPIe|Ve% zDOm%XdZKf6G-RW7J#o<0DK9O_+Tg}B-y=TY(aMVUvt3Oo=5Hc5iE_$vK<+V)Dx5R0 z0s!0BX=Pw~k8whZxl+VbakCOhR(t4V$SNalPg9?7Y#O0*3g1RzVhn_<0Dl~cw7VSA zsm-e`Z^UHhCpi9<71904jd&k-t~sl5vCkRZ=r$uHgKLC#Y zPY1V9KN@RAG>2~9+mvG&6-oCpX+MP($)%Pv6K!l~+KYRH% zG=@(BJ3kt=r`bz$0+`o1CaESwnom)ynzGiC#N}NIvh@a?nKwgYa*Qctjg(}m27zeK zheq)XmIVVbc+~FLPZ_RxLmmY?lHzqy>UQy8i>TyhZQH?azN8xFcF35`K4Kj-&f5FN z*Ed2KqQnau9QkJftB|%*@&IVKC`BXC(Y!a}ONl1GjjeBIfnobde1*aL*#7sua&TE( zNxZ|zL6gsFO8o)y^*seI!e2{kZ*a3jv6%MG!Nj;6ZYQp5in50OD~Q6Y2HlnV)q0t< zlF^*K^}MiOPYm&_b1a8^YC#HfoDXhm)n>AXRIvLE{*eqver1-_Do7YMdYC?S))Wp)J?I)?#3Np zt?sOJOGY3&cAtOm3NE{n)Vf5yK7*=@K{~^4 zZ0vBcO(y0o>Bj>dYax7AT1eV^b0FXQ&zRtkQSU|2a?f*WD_HI4k20Yv{;=os#Zi4r zL%R}59M24a@TkK9m(+iSJCVxzm30=0ZX3xtUe!`-yAeLdn6SY=@n<#h{(!#;;zGD*N}P8#NkgXxDJ00!izC;M)otE-Nrjr>v&h~ zQbKafsgg0$i#85P6m2aU;^|{283&V7?V3x(NY&CHIXLf4El16z$l4Pe#d*MNX0$J? z8pOhU(I=bF(w(e2byW0BwRn!UQEuqV^3}VGW|vJ_UBdZ50KT=9t41>AzNbo|IY*k> z8d@Eb&t)4G+OV(8aqC+)wu?4^t!p^JZJjvE6rqOp&qi|bm{)5sC5geUm^5fbrQw)vk`h25@mo11D;`}ca;r(oHnHYD zCboCHy15zJBu?1IK_B5;`tF}B`g!tHH<*Mrdt>JR0NH9M65%xCV}L=G=Y!t1w5>bs z`fjPQcS&xM%(xuj7Tu4+h^-ASSn&6VKeN-@izGH5qYK|GQn z4Ft;^iPsy3RkP@NepH+zBGYGXj|JQT(UMr7kU_{bfu`tMm7U(4ZKQajzlmgbBPh8i zJBQE@&ass*GL_b-=|1X!Itt_UuM$sfJJ{SXfssLfDEPlxx(!(m;JYY*9UzlKEBA?+H^c8ZGlF-r- zN$aUb!^2HI*!|uhiZDwN_r3A_hkwSs6GojNNQ`#EI2+~?0lO!$W9{!;>M~llbEVZ+ zvW=NbN71frEm~`+Bv~A@fXIGQeS2pB4{TRsa9wRY$gWv{?Zs4aA7;hIt7*jJbrpFT zLlAJd6{Brnx`oV}z{-rCpr6LBZ9*pS>&dR7y+Fe-ZOW4gy<2*rm9Qs2> zE(7u~yR%g6{4;rdrd;YORUm;$%XMNZBSk`2x#{9FNmr!d$d{SV%ObpEcH&gpfpraDWLVG4BhPPoZc(}B`$VH;k3CV+n6FlIG|Na9qDQ7lc?zoLs`Ns~WovzyQ9cqDl2; z^yZWIV%V%+SKLkqsTAg6)4n;u=~>z+-t`{Z3@Qr*OtS&WBc(O2Ax@l{q|p+Lho!t_ z^U&k2DnGU`+)@@9{w)6hN*i4el9m1WXfk^^ix-YxE>-BmjQ#?e*Gq$m2q&m3O3kfG z=8Ae0+F3m04S^(Qx$B-d^`z7XnF`9Iq3RBDDOsYFl(a2MfK41~Mn}%XpTqf@t!pV@ z<++recuE?5-W#J$|aQQvKTB$3O`?bz-$sL6@iH!R#2TOD3I2hn|9jS^ixXAwN#OBFpkwAZDi4EPYXGcHXs5(3=ghBs@58rMpFnGEB9Ddrz=DCYf1jlS7zm2HxBh2@?16) z{J*_EO7v2-nqcfT0q;mLQN_*w21kG z&MUc-y8379TJLdXuautNZ?e2(g_bg&O84AU(%k4ml~2J)ci*!vbDXECB4LQd2%u?;L5#@ zd*pWNDVH88wA7<@zR{p9D(yEhoJbCO^yJgHC|h(=w%7H4wY=+i!Q?q%v=#gmj0$C} zG0%Du*$S($R0NaPiiayLO;i)o$9LfkGVffFUFveQ@WkJ{58V3wKd;idEeTfI7+{Yg zC=LJ}`)0m>6^@-p%NJyO)Nm1{9%jAbr;oL!)3k{6$Xo}y1)QkHBlP@F=nZl{Ke;n_ zlIcg>hipha08#x(uG|(KMhb6saAIn=ClqpbGZtf2rZwrDL{4)}&oqiiOmSP5mYQCv zYZOZ`vAT=|JN{uQKu}Kny5CkFm~3R_Kkg~MGTiv`D-$7UO^yx`gZ)q zQfZRYTcaqi=T>Bek&3z%VZl-`N7jK5+uF@{aUHxXx=C0nk8@pjguFQw)x5fm{k)A3 z;?L*vK(~5#y=|Tue6;rRGuLBQ!-l*-c37_?g1G49T zcS$2J!*He41*3TpVNWbB&Rm|P6TkzGob%eW)(cB}CbiRAX>H7f;bwf8$sB+?4|>^N zc3loimZ5h3FPWEgZS>pOJg9>V6+53Ms!eY97lJ+8-bUqsCmyE*)2(XVjwmi|9-`^i zmzu_)wwi2NVhaFI2LrjTkHF?z9}yzkxRwZnf=Q7^);Rfn^NiO#>hi0ejS3f)p5{&O zh4t8q{i4vHFS@pC*d1ZKGPJ`bqwU*@${46re6e?Ddc#tLIj3}Zz5MR6st`v25mq($ zRT&7*Se~`Nvy7eEIQ^`aiRfr)SIrfdlnV{(#(1vdP0}>`?+jXth*Y}5Hn<#viszIe zDc(zCsxGpHqq8CK0{;NV;r{?=e=3{6@$CC@uOVf|YH@Du6NXlc^)@u=1bW@u5s`|g z_K6_W?tHMMla6bqlsTxy9*!=ZS>aNouI>y$gHt>Vx#FwEc!tvPWMx;8kIPptjUHAN zIANrvzNRwTnXROaMtiTZ^sRj|CzVW5#l4TGTDeIr&KSl`=y84%(&w_0JL`DNqb&{1 zzwt7Ijmz!`>FHhL2z)p%E}fNGe|soX&f)+8ez@d*HH()w!j9C`UFG)-n$JtK(;Djf zNY>lS^1$lFM@9TQ1IYHTHt`>brPS`EytyhBbG1}pApXBvJ)7Q4e%-wyjU8vjt#GCo zV;1nQ3J2cX?t539-rmb}S+Z6#epSz3b5hh+$fZ%Gs|{ZGYFEB`91Lv(&Ta-elBn3}>PNqC&Z;nMiAgQVY%CVn_s**|(CQCF z_7$}Sv@!DI3F+L{>Y7F_saa&WcN^K+y968$=}*6Qk}oFEd46HRZ1M+uQ?-VteOc9L zJ|om2NUfoi51VO~mwwZpoL3iXr(S7y32|-noaC!HPvz8CJxVfq#T|69RQ?$~4_&&s z)OHq@`K37L_b34TF`w45JUM%LeW#baSdqhRQjfSFKzJCrnqVaNlx&vW@wpw~9xUM)h&5qK)vDH`X@jawMTMrv;aE+x4iO7uCs z-4C0_Me`%2)L@1`4PPW^shHbM3?2eTrjxlQb>5`t*tYYShfYyR(Qj^eAu{_A44QeirE_w=f zCz%Jq_h8;#LYE33a>52m8auY8saH-p^Uri!q1YmM8Qz-T|g;bxs zWxG75kW^>r1!|ohQ;+efRMIq-DZP=95Gx;C_T*O2_B-ToKRUG(j3ri5+{f6&Zmp)$ zogVW~41qh~yfbgf6SIO$5ob2A^x$>y}pq4c7` z%x4(zD(SZ)wH6)9K3GxBPc$q?6s$DLPk~Qp8`7k-(OJ2vOe9phsUEAt`_cRN|*y1+At}w@SW4E^}`n zE%fAZ{{Ysf?=n5aKGVdP*4FJ0o{U%&D(jr7?0suzPM!;WGwhMZjKR9RYD*8VS_w5_Fyxj7hCoA@4H(06M9oU454J3xU=t75@Mun##0XskpAj zlw_k(NuJB4yir8yD8#DUN4LFNU$V^6UP8b#kXR0fJ?rJ3=g~?^YUh^t;s%Svcfj%s zh6mIY2D*QVdO6>3sl|pzK@@}zgni%BBk-@F!c|hHqtzZ2cs^#&D3oB;i-@qsyN+uV zao)5JmvSuPyAke=QbL5wFvn6r=~H!k6=!Rd7diAbp*E3yZ3Gj|X)48Vf#Q9nhX=g>bW`D zjO3m=dQ>>4(AqU5v~TY_ty@a4x{O@kLmbU2Vky~v>bE0|jN>_}FRkt_G>GnXD5JNZ zAR>90*n^i#^waut2(f4ADf7vT_ByrqXhgUh0MO?BFeR0Nf{OGZ~x+ZTX zZP2d*Rc;s5=cl!5n@HrT4tJ{1xHXfZ>7vaVdx;6+S5f=02LKO2{c8tPzLA^&HkV(R zkbgYXOI^+A#X&T`F;-bFZxihDg6!jJ1I|ysYO4$@Z2RGqh9_}sjGTH?QL&q<+jnN& zWFFh5LwRg0y@vRuU|UCIgEn|UIWy{?LZx-6eB zAXPX*e3LSgT6N#00|-I#BNju2b2CS4JwQ zHSf@e%29P7WyuV4TK8fZ?ckaa0e+|4p7kx9Zq429%5rOVdJdlj?x7CIm5QW^lE>8J zzg%Q<&1p%e+g=MNyn#yF{{W~$GIBqN);dYv#VK+|*EyX>TGF)-wOzDO{T^x~$TAZ$OlWxZ~tW46nz@-sR7_RP5 zrxlrhdT(wXNKm3@GVRq*7#*6gbJ@aD9eSJJ4Ok1;)}M2%M{I6W!~!~ZIuJ%Y<2gj(SVOCGyTv%N@9j4nIuSmr=@=meBR%QZm=6 zv8lD)pd*Q6T#lr0pC6kGVo=DTQnS=~xDN0p1APH8@CKfKqtyNZ22<|t%FE~R)q>y4hq zeFnzpL8xbXZu`0Crxfbat)rxu%+7>q(QXx+yX=Zqz8AB!zH4n_?HLR^q5<{;&p($H z$Z8iiSBxaNo=KPupt;TkaXNM98%}?5=|)r{{n(hRbl=}Ko*dVG&nJ}EOT+#1S&GNj zi;I-nOT+#1Ss|O&n5=H4=le#n(pDYl0M|e$za84>ZVTlda%&sTKD4Y-n_UsOH3&yk zKb>L;X>y}QUTdQJk-oZ(w|Lw0tT8b?NCW9p$c>gVqS;6P00Bn#+w9L+tNX<9wHo+i_aYw=-A5!bq6Pj*UR%{m=;a2q=)hk%WP2W>z?Mv!^I>|G8)e>v{sCxdjj4pxut`Fl% z&!#%2;W|QfH-dluHdV)iU(1K2ld9nHA^ltR{ZLK9`RflVI&h8CbOvX}(d4k-@DqMb)hxzk!Evm3I6s+?qk2>nM&^ZO`nZl@7LAr5lf zzB7;OOGT*p8_^wRv*GA9gD#<`W;>DrdqzKe@G=QF9Coami#w<^VQ^mM87641H@O3M zWgesflZr{E>^@0FT+6-E?=rU zPfw`+RlwJi*q$!?NH>^#V%2N(eI+3W{;;KX4m)%Uh%WeiP9dM%G{y|IQ{7PGp< zkxZpoPk(XfYnjk~B3a#{+UvNEWH=WnGK_srI(>S5E6$b{wI_HTD!eCtV~z1Yg)X#d z$s^6SbNi{(aNftz`&X-ZmU@I_BgH+ug9iXQF!vR#G4qC%5fZ~jO7UllSnC&>WwY7@ zSB1I*!Ay`_1arRo9kniIKg!rR;un#EPg_a~;GCdmbq!t#))c zNjpUu#vyMvnptF6$@{?Ip2I%$x3=;`&GRuEv9ZQM^dhMUNmT>5nVD!Hy;(p?BraARnsjJX2?6?HA(jnUP&h9Cl0jmM>E zr7nh2jWpAfv6@ST5=g;V@&hOY8qmJeW{Cv8U?>=&QaMx9bLsy8)}*9V>8_7MHsV=+ zMdYv?di`@!U4tA6@~-I#9C!5})~$QTEZU{eh4-@oJjJ5VLV6liS7tLyw*+u8)~ZRZ z%;7aR?vAsSt*-w7a?CIe)9q0KP?yK z;hUvslP8y|B)3nUN7AjsWpKbU>?{uhaOSCLt1^}3?2%>`D|D=Lg>ZQGsGyKEiUgy{P*P3|0#EYy& z@@5-&#~JEL>VLwzBLx(2xjD$;H8=ZY&vPSuyNE!4i`3P5<@2P@HmCtrf4T=jS<6;$ z9T6N(s7Ym0=o+WfESCE5M1%Kf^FPzm@ud{o&~ma`n|>M7*4O?fQZ8mA5C-Hg`(NQ& z%FBQU01N@?T@i$0a?Yfs=uQb>Nhc?kVtUnSCgT9%M^jgGB^JmojCU&}2Oyq540Zlh zpQv9*5Q}g}tb~6m&9kC0vK=06?Uh^aDxCdluJ#tO43m}TJ@RTUYNK;X-NMFcPeHjUvGS={-D$RDf^!gTjYC>Z}Xbvo~p?7V6}JL?S9P;qk9}vN+6GJ zH&%YBaJILutjwEOk&bvhC`#x~4@)(ve@B5vuW?|qEU)cM|QD6dl}+z?#*<@0h|fR^>SXO$LjYSea?6c&Bl4 zld0mEhKq}om1z*sai|aJN(N}=EmC3*qaA7ca#-LFJ5kJ6uwza~6*rW8u)t@Eo^^9q zgp!Jtv?IsMox|xwtsFR4a0%!r=eaV&xUEGQs`B4M4tD(PIss3ZUd1ihGaq`*mgeGM zKv1K&S~-by&|glHVHD>alTZkH?_vq|CX;s>idzn~y7GZm8B%k{YEcreAY_kPN!V^F zkGZ(qHX-1SO+d@@5?Y4a5{g4IM|yDBgk)sWC`4AsBA!{Bg#ZshKnP5y__+Gg5`c0^ zY5;n>FY&EL6C*o^;Y>uHW)CuC{Wz$~s+<$J`%o4`{N^VE2k@qWx0TLu)bT)B4)YG} zrAR&VND1dBB>K=6Lb2x<&(fN6)a2u*GzF2=Xu1i$(9PhySlKrcj=T&L#&h1Zydgch z=q4M8{K(}l8G9aUSZs0DcA0|&zcAcRE4Tjugkwm(v&E!FDN69cix7QC&p+0wyBVuX zLlP;XxDmujB?km|tq=G{ZiStZwUQ=~xsFyvIb8HD-x(PEXjLP74p?f6DpuU7ZDVTp zxR%v!B7B7k!9tvMImh#@y+232(r(@x$e{|Sd~-Br6{$a=)$CDs{70x zGs4HtdGBrncGmLkCOnptjO7ofBc3|(TbG)J&9ugQi=~~icq_vFaoZJ(oSM+|VM3ap zyzXm87eKH`t!^$aBb(+`c){2RJQ3Us_vh5tGbXXReUTFm2sltmk?uV?tX=n!+X%;1 z6{9;H6I+e$c1Qyw%kpjch#kop;CAC3Yn!sZnf&RR3`n>t>YEitIQKclWy;&o=yHcG zq;22$A4od1lDxMpE?LQ3f)_mQ2d^IeYQ2`PGQjsicLzIJ!OqY+9DRSSQ>itf&m0{# zXk7Sv&d{0-J{GdP5h#PqkwXwOkXr?Z8OZNd?X^3kyDfIoCj1!R?(WFruOH-8skH7~ zuxZ0b_Z*XWa`#QQo=rjrfs`shgZSOP=m(+y06ii77?bAc3i=Vp z*0fdb^*LS!+9q%}npUNv>5*Pd6~&~?#7GN;!sGx(4ttK7{5n@{t?GJhlvYt5b{dPRk<5s?W$2HBvV}i0s$Mhzu-_Dme zlFpmLO5~uC_b?rSJ;)qZ>MOCx%PREM_9VAQ|U*^f?&*xvpj= zQ6h<&K@mSVjPggec%i44Chok`Pq{nIsheWaN>b{=H|# z;@fLY8MONwYnV2Y@e<*HbJsaNKhJv4miltv)ai{&UR?Hlk}t!p>T^#N)|z6+5C$p5 z#s>CLlDn~xalpu~Zhdz05f82#%Z1IhzGDLi&OgPv;%#?rDo@b!7%=TZLv zUn3V#hS*LfxsjxOxe7msj)Z~5R=KmhnpX1cWK~2ALXFuOz(321-kl`(I58N>UeI5G zTuCk7EtUY1DZ?V1jQWpkR)(pg&8J6ia}f(Klsw48679xL200@=z3IWIwqsdNaZXqC z{{SM!p>OsL7TRfHA<;o0TR9!uKA7kada17)ne_N1R0(f0M+}me$pql;032=nxvb>h zHm6iMRWzg1$ZOgnt4u_5sy5N{XP&ioSz&csS7WnyRg@>n&yX-k_0DOhC9vh}-^Xk8 zGK@`drOII=0|7=q16SqKWQOKDSTdp5`HaLcQT5NIQcgTw}}$tsRpYG-XlqFL-$IW>)ggPU`T7xCoxPm>Z&$U$aBhsX>un?41EKyr<1lLUIHofLyPZJdF1dj67NiJ{g z{Of4oNMbA%58vsG(py{H!jS3{sg5$s5DnC;ah!JN70U`%x<%Ei7^|a-_=j?Cd`35Q zNPy$(Sw1VcX}nE7?yK^jt_Eweisg-&#a_+b9JaRmY~voXe@c&EZ{0+{?VsgUQRqc9 zWdX)%v9rq}Ko^HQayw?Tx-XTpdq#uFvO<3JzdL<9e~;r*NpiOMjy2sQ50!fX>T6go zR7CG5p?RhVpdP279ZgqAcBpqHq;I^(`F$$x%&dp9yOPsjAn1eo(t=9h^q@5<>M}E0 zFd#4l9-V1!Mk6cgz<#uzrK=*ec@tZ!FB?H4(+4#pkMxybJC*)bCW`haw8$+X>Uw`u zQT)P9sh^Z!VyMWzyQFZr&#gyqD1?PSG>4Y<_M*$EuKvjTpWZu>_l8d~yJwI(3 z=^f4g01|1-J4GtC3uZPxoyYa9JJ_0QcST$ttMwfzbJ|y1FpVt|XA5(ou$OA=V7>9y zw_}hqF?G&*_O6&M461V8=~(9(ZJ~iu(pQdD`^R?-ezin0hq+zKf++}H&4Osu!$EJf zTbtkXfhzp}05SCjwY(+bJugnOx`$SQr!0O(192YJjcin*Xs9+gVUzc{k1odPZ zrPQO2BCGQ-9gYX*Scw=7k23)G?MXo=a_5b!IPz&@N_*&TK*A!NXM3K9+ZCQd=Vwqm zfmEWAjav08d|#3z5-#o)Pqrw|{o<5xhC;_Pv_8ys_6;NBb?q~rF80x?d=8S{RPd@Yj zZH+n|%}i!h;0&L0Ko2UM5&-;Zp^$eOpao|Ndk#1Pnp|grpLzgrewY~g3QXr5;(!K~ ziQ1rn-`<0c#EJlD$o8ZxC>X3Ta1MRx2{_FlI$Z*0j>Q+YL~_A$ zF`OUaKDDe~hcx4P9Zs34$fYxe3)hZ+{e3GU_fb@g!5mu^M$`o0oO8+VPeck1Tbg$I z^!D<-q-~KQAdoux4EpxsxzmW)25hr*1aJ@asByW|3ERxrpIDvuLFNdWJ`PTL^~H1d zGNVh+%3}xSBdNtp(?eD7`^?mqa#>xj7HoC*6-ql+jZ@{`SY>d1dFxA>g_ zWsS?S2aujma6L2lRFP@&PXWma*~U8`#8f#+Y;PKHmaK5PlJ&D2x`YQL^#}g|)tXy- zDIvW}Ddt$d(07S2G6n$u04Ls~SJc+=huOO&SzRHK7C;mczayS-IrOU*HZhy2u8w7k zc}S&WjrbkM;rdoh+hch9-x9Z}qi=VzGPj?<1O-Ce73ueXPHME8YF}!PB$7H>#)P0i|Iij6ukN-16UIBjO?!s=qyc1v!-$YzY; z9Cjglde-#b6}8kYW4_axTPQrY^AhD5iE+*szi>Yc^Ib|&N0#Z%O6gzFezn_OwU(Ku{hHp__YygMmgJ64bM?n#jtyq(H}IWw@JS}> zE1Z4DhP4X6o(vi1a~BM{3}tk`7f^_UpXVSVvIpJEW@`$$LU)sO4n@U zt40cZuuM-2t-{@ZXS2J(CVaLZyDQEMfO#VW81=<;5Xg~2A)f9exMeDh87#OYg&jI` zj1NlArA}qBwjU2T&gxFxtY_*M6Iw`-GbGT&qhjz`KnD%=?M=9v#Lf13U1V&9X-~*d z1I`Ci>-g3wNc1J_n^#8{sol$IrM{oANn<3c@t*PW4#Vx&AC+w0Xj*%j9((ZwGXkoL zo7sEkuN_BUO0SWG(sxm0dUX18$-OO*#-y0li6ht78T9nxvJ6JlG%0Yi1Z<=6k~?SL z7_Dyloazy_#@?-}v%i}pAre!Oju`jB?^qMe?tx}v*dHi7;Qs)JS`x8M+_KjYHP}fZ z1sXr~VE+KMp1+UlS$4&yx)ThD;>rWqlh&>u5zjrhJG~(-Uq!qB07Zr~x8=f&)g3Zu zQ%#ge$8j7A#UsWUP-z0j@Ry^r9bEMNH5ZE+hy*o|Q-^y0G zEQ;L_46}U&ON!>W{jH$lX=xH?RJMOFa-)LWWL2wuCTmnni5lY39kQxWoB`0k*V3Hi znz4;4a!%(_t6U}Cois7X9hq_nImS8umB+5KG~0=6-z^g>Z1nabuyA|Dop4ZJ7N;4j zDQS)Ltasc zdgIiZ*FDQo9xP5w4-46}a?;9xa54bLVeL^X zTSIzkS6s();LBi_&yK38pT)END*ph2ArB0fIQ#bsgFb*UuQshmv)uI3jD8cYmfsj| z2tDdcxr8=S=c^1S`^SM?%2K_Cr0#J3YP7R4{wHETGghspv-4%KyND@C8CFr(f=4y$ zQFTx<=f|tVKP2)>N79Z| zCsCuHI$iHSx*&hNE2wy|v~rW}S;?CVq5l9Z{{T^2(no_&l+jG*&5h6ZKT%s-DRPgP z#QO!${PX=qZiMEJL>%TH_I=s^0G3~Fsk4mj%Ayj*QqD>61*6H(V;7 zgB*6E_EsEK{o-c0kOoF{bzV00H7rnD+r|uuzG8B}Hdpbbc4_XA89;BC=04=~slc`< z2g(erR0kV+^f~F?rM762NO?gbMsP3=eo@Kv_N5mq1cD2N*zx?SxZp7Q^vLFwicKU&%KNTAB6ax3$YN4IfPO{6(c z(zHxVZB|wJl%C+{uf0zt)`dFmn@R1lUt5q}BTJV@MnJeY;f@dV#Z}eeyRt~FCYW3- ztR5MU4yQc*MQ3Xo&Z3o-t>ko4=E~U=>>goMRP!;CM_@YksgoM;F)nm35!p%QeVvp@05U|t zt92j~?SopEy0({fXAQBrxDBupVp3NlkgPiAuTSrDTS8KM$M85SLX=;-JN(h3rRsLq z?wVL|*F+ugMuf#F3U>x1cJ<9SOR$d0?&DLA*Y~?hE1}yN%H-`Gk3fEe=bFY!3O8Gw z6rEa=QI(oq&B^7B9wU7rE)|gB@{$9|0dv=my}ugfX1mlcuY=7K2wF2CkyLBoNRz1Zff1JE_2-On)2&Z0 znWr;e>3>+(qf;H5#BK_O`NMkqaqnF1!WiyC#S4~J!zm!{^&yX6YSCy!nwOWEr)_cO zY6&g$M8h1QY;s9ot~vZ`IT>zz*J;3(E+2Toz&QNz&0X8v=d^9^od&0YhgMm%&#H&4}U{VEezF8S7)&u`?#GU^9q*8 zlLxo2^QXaX_-W^MXGydVOdv0po*$P%xJ3Aj0XjU=r5zQp{>e2C<;b znqU>bfe6|=;<$<%Z$L>ck6k|%bc15?dG}2)1yR}(N3r_Xk%h_R;Cj%LDZ8D$=ZiG^ zxW0RdB#}?bS^}i{q>!NpuN3SrYH|BHt%-CymAf$_<}!GG zzTHCXl+2OH zYD^b3Q&yXFtfSVqj83|7xs`J;ITZ`3EzN6VtwkY;9(k*AMUh2>Izof^!sKE^RHAH+DN zV$|Dvnvg-KY4^m7HrceWb|9_*&fa=(E0nsuo>q2e+2>$?IcG>^jyH?vSL77uaP~DEK^I+- zAO$03+{ixUY)@axx}4mfB#$mMqX_$0KSafgO`6~=il#-+&Lbmoj{F+xbnQD$+ZEih z%Fhe=gvNG>6k{NI08b~{l@Xxh6L(GFa!H zx$j&Ix{kpbD!I{lh3@02b0Q+f+bOa1_w@mO6Lx>hG=Dq zQyPLfLU!l93N117Q)=B&uYV*Wbcz^JSvs*|ShAZHJ_5!Fv4sr;%-n?tDiSCJI6x{3Zvj-kQN4m;9lx^*O;#(t}9aSgjj zs!4)X+~bwbdgt+~x~8+JM{PEnbf_KQad^%V*XjLeNs(5pWv0YhR-5+ICFaK{vS0uQ z2dOou;rW^yD`2v!1<574F5}H<%K9Aflve8Z9?+@@p`qWtJa#qm7)dXU~Pm>H7VEs?|AB8tjVzzR`&Ka_$fEe$`^{r9KCSwDN z0IM<{FljN4l)!e*;+(C3Mgh-%N@Uz=LI6A*`WlpTPQh{@DpHhKR~kQBa9mTcu1Os# zan_xLILEk(vl{0I6znUEOLp0UK3wLm>@1;kjL;V|ifijh+_Ru?eUO0tpWn7N3Ab$UBu z-Kho2o>|3et%DKwO#yaeG-kBrvn$$wyD~~+wOW!KP&<@LRL+>BG|I(lM_N`Y<#H-b znW<=ZHkEP3K^&%CTgw2Fse%XRn&-ZwacVa;t>D_Aj;F0GEYXmsk(#t+a&U(=H(mt+ za(j5Q#Vk^6z@{cS=8oCM_QrUpO{c+i8_LQ<;~|0ROj>m%xsKld08V9_l+VgR=zCT* zj;G{9Exf2CjrwP`7auL2hJDqW+PIC#Vn#t7DpzKi!Db`@ftn2yWe19`G+xvcq-SjF zR$4#3Or;`_8(0z48j!CvSf6sueLw1Rkd=aGy0;kqnQHNbOpq zsXHR9Fm637V$#p26q_bh3dw87O4>_KI#gWC+8HZt?OOuZ+Lgu5a{86qSxjNZG64ko zRGR9Hb}TAfR)DxtOsS9fk*5O&UgaE{n^DN0Qz(hy=p~P&m(}QnsEu ziNrZ%k}(6TIbU7lku}Ec= z0V9ydia8k)#jk~GW9gbMp?{^%a4lz!;C;k06m0k4b*`N2H&*lEC)~LCSvq6-Qgr(g zu`R1Lp~^MYpqguY!zxOPhr}$qSD^$N^cZa{b^B>1yhRg)+R}~*2iMlHX-9LqIGU2! z@(XQR`@+-QTiDF66wA69KJdrzu=gD+cgLO!T_R8I9Wn;{I~~p=A0m74)846a$i=xu z>7q1zE2nAGXuf6PGG1Ls)49$MJ-=N2b6DCH{NG?j^OwyIe7KMjftD z!&Hh@H#Oy3714?0SF62UG2ip3YSU804_HA=kXj?(bI#@8a9$Rdb2MCcgA{tb*HkjyJW{Qk2fqKf9t>e zF_U4V#c?!q-bg@wCEaEVdhIKC;~;tPr%g4sOcR_f}Lq| zRrgn9{{YC!)zZclDGV!Y(Ix`7ak%n5@lxAqS6Y<*b>{7nN@Jqo5l-o z>$%RuWp#EXyH>}OfxB;9k52W}!5lgqHHiWJHAJzF{3-7-0c+J@*zLFP#x z%ahJe9e$>?M9|t9665_Kqkiuo`D!{62d_@K#Xe`ynNx-A_c@7v+JPjN11d&xB?APl zbs|f9sKv5gMI>rILJs1LKZN(^pVq6I&s1cosqG^s8~Cnb5ydyjoxup@e!l%Zsx3!E z@?>j^e6Z&xFjOCLPVr)X)*F;{b@T1kR$rML3f`l=HsbSkkrYG*c#hGY!_-w6N?g&@ zv8KFDXAqO181(P_=&*prro$#PxE97b)sq|Ymp;=PzX1HFJG=F)SG!2<%f~9bvhvH0 z6Y1|ynW7}4EydL?(e5rIRr}>|dng>^^UY9IjiYHAOvq0P0sP6Q9m$g#1}XR%;2e9_ zhIbKh&R0Bo^rrz%!nlA_!s4BT3z1I(orQ7A*#I8YWAmqAxb2zgLgZ7iHvlkdQG-sx zxjYOy8f;irqQbd(P?8AdrM6+5@CUss1;-eOP1wQp#U$e7XB>m?O(8Z@e5pS#;%Iez z{PlJw;}vq)ii5GiENe#w;H!gI#}%(g>_$i9p_1wmq?r^YpHoHDxUs%UYaasyu@3 z4Wx`#?D{Oyw)c&~o<8<>*;cWNECP(M$F(KR?&5j;scJi;F5S#}zslZUI5=_gk4@bwoxGk*>oh94 z3<`tD2Dq=O*4MjZyi0tdvhoPv=QV#t5?*QX%Q9_|xg!VqimqeIsN5xHK@@G=R;frL zxr*7C31wDfTy+B({&h-Z^U15DHIr*7$3C@dP?N}&VO0c?l4_H=r8eY?7Mo0aTcN2e zc1Z~s!wyHSS}CJ9Q_$4MCBzGs9EBZgLR;`4VuO*urX?nhUMNm*MhLAapl5b=>c=LF zl;%*9916X04!~$bjEq)~hBalA_e^SyFJmaLHDk->oriHOr;^UY&$Pqh9T8?iJk%dB@0#OujY4fQVDXc&zViClET~!DCp76<-X-U^+a}UE2CYv# zS1cxvzfn}KLTwLAE%E0at8+_$Tv~<$JW_QmCpL110~F+Aq0MY+_FhH8zBq0vsVO1z zT%5zU6p(Zxu35+SohKi}qOo$bvoSfHRkt=OTbK*Vc<)M{tMXB4xt1}(T9wQ0^ z1vnMX+1}v@BX_Q8*%`Mh*zavG2Cbz{rn@w0G-Vn^JdFKoD$iK8iY1j_$&fE9pnTu` zddhH5ZOy68$9;)>J>d`dN1o=zCEKmL;|;>G?0xH=N}|=B zFqJo@fa%C*^W(%#ARc0j;lB>wT6;X#+I`bN*|=0Y5l2()Q_j=hQ&$MM+3Z+oZDT}F zDme=eoruk1SZaEMNIuu6&kTWb&cFo+w;jJ)rlq@gGKzv$iqPD>MHbP*SmAM;3=@un zKU&HD)RA3*!7O^3$;@^VmuwiKpf|R z$F(Gs7N)f-^R>yLG#X9B@UkRnwk#4@^f>M7-nSGYA-A=aNk4qJ!9V?BTc@^%oh$0O zo~JQ3fi=-c-ro#I%Ia~q{scN2>NOeXh6sMn+hRo<%8sYFquM8RVS5QhDRVQEj?P^l zQu`hAysIf5T&>7bMo2veUgD+I?$-V$S1p|NI5?#kNjsX#tYscnb6zOZqqTLK00eRz zkevakCvpjf=5}b=P66xn2l2&B9mK zWON?&V^LLWc^C|2j1R`DDUHrAPNpaoAX>WIBbS+42Y-hP`u*6)}{jQ&Ipb*gurb!7HhHnU(={>uLVdkeJl_4ciM4;WebW9@UKPn>Ou2^jj;GL>GjHBqI~M!k&M zw2^L`%7_DoW*KjsrSSx|zD~p#K07c#17Vfo}_@I70+HZ5T#78M(?vdfU%(NzND5jw?=S z>}|jisb>1qyXXqtSxGdYIgx{XDYNP_HwZS0R*^}>P*yuCT zxf@MNYmIh8Bq74DmI55?8TB2%8mQR9G&xny*+VU%lHF9u6z46Bk8gUXs9g(vE-3>Y zx!A#abB+ych-u!)rD$6P2{=7{>ST)!q*XL#*{Nx3xMvl!q}p${&O28e>L{}o&SzAQ zW<_4tpxbg66}=~Qb6RZY8ZX?RdeCcp#nZKGb5|`)RfLx5Tl#*K@`BrVp~)dSi5O3A zMh+{Zy3)!iJ83DVB~=*F#hArjSayQ6?l&iB_rDkBUS>rod7;-)9iOE!UAzyZb#CCzK%Yn!_skKQ&wh_ggyQrz8 z+L2W(wkmji35k@B2&=YHEtQ#O;E_tCuJkiq*@q>k`ecI~2CGs}98Au{WSrLO_92L4 zSd>;x_NZKtm2sseH3rKUm#Z3(4Ri9mlAcMel^*4&33qtX6N5|^%-yS11dS3%rp21H z7Q~*Shh#gWo4m|d%e#*-0KVqB8;f{C0O5B=atRqCvvWYuG-7}cd zH1{rys7`+PYUph=+pBAZHw6S?hXDQY?MJ$nf^Kcu=woZTGwJZP+-3G-CNQOl{Cz97 zZ9`3--J`i(rE&nzZ%oz=yXua(^4m!sd7|nH$IRXrJPexXE_59xKO||eZDJ1ABZtm- zZr_bWt!<59+Iy1ez91G1b~fXT6IiVngHSf_118ay=R19?nrUw&bVjQ9ZgrO$mBy#0 zx(ou{8ICbr{oJ=}ViMG&s600AKRQsAIJ>4)l9YAZ$d32z_Hmh6k>dv}P?p7EjaK^3 z8o{WvT4!DwlxavC-EO)S8ZoWysK!W+M|~l>+4O~E0Y%| zrOTFgGo`$+FP9uv(riu2amQSG^I6ltS|j`Ymg?+Ll;w{nl1~|~Sz;-txz`F&wcW`s zCcHNuYRL?zKf#cH8pW4ZwzJ%^E)F*CJ!ySnZCVJKQl))O+r3?dkd95exg;Utxm!5z zb%~0?tRXFil&K(lan#m-yl(6?ttK3?^!%eV3B>XGN-^1)P z#($MuROP&#g*ZYT70R>T3#pP8i3knGI~vZlhFB*T4hUGAw9Qs z-<3CywQ#xPk7~VlV)Mxx#)`^g<@fx(c*o(2)-#Q_B5LvrTlrSnHQm4)N46@pyb?t- zE3N|j)8$Rx=5NfoCB%|^uD>gHr%21>L))uF0FQsw7aJjHe;%8nH0j(zE;bW70B`#TQVt5K3U$UJwdxV0Fz zG8hbi62XAl267L2)3n;^(H0=LU~!LHo^40Vp^g%@WweQ8A$ogutrJFVGc+w<7Hggx z6z+-CrmO8*3t&%2U&fu#wJN6-aM`-_!TM9sme~?HUYu2ktsTw=2m8bdG!@wB?f~SB zkyqvX=ONAC*-`ld3O{3)iwc<^2qPtMbuO$C zYMjniKuvc^q-i$sW6ARdI>>s|ePTU}R3+T=*IMri>2UqaXb}GZx+wPRu+<0L^VdRs z>$;Na%pHnUahwGddv788A2ZL|7CjW7psv2^?$QP#ON3HK${PbT{jZRFFEfdFi0nml zDSK^ltt#85It|>M_ovU~KWFB7#^Jyt2el%S4>iDg1l)ONJ*moa4{j+EM5pHe02+J5 zV^nUtg;*0uBS(@L^Y7R3sH6Fcu5b%-4Ir`AYZ{u#;oBAt1d=}Op!Ngi{Ej*Itozs^ zeInI2AeC1!rg4B<=05)bjcLtwV?eCOwTZ24?Tj(6nI<{M zYL8C1wueB9_DnNeHU>RwHvQ{vWh!z??1ty}u%D9#t_!<)uVc4Z{{H|Q&rH)&aa`6R zMV$P0Y-RwCrn+X+@OI-6Hv}1EPoNY=g+%TtPu=B4j4AUg%yYDHSw{4p zrJVW>1#9eN%q+{gmGdwmcu~@`C21z$j1NIt!itXLu|{bnnU5zGekfE9MRZ;(VvNCR z&D2($sxikDa|PK_8znrNn|Mz2n`|`{!xUq+MI_7FN&3|Cu(KH?ioFFu>{_WrX&m;k zc*&|VgO8DGH^dLST5%f#h}SqjT9>0YjC3shr*dJMvDS76Yj9~cT7k12*UgH}Nr>cj zsBF?PM?oFyK%0}3-m#Y9nSTyxMYL4o86#3{QKlFzSh7ij#a}U!q;yvL%S4zgPB<0J z{h<2Pxgymzby927N=MFk>CJO7TyipLbB)rz=U)}?+9Hh^xebzQ4$kXrp|izJ?#oeW z9-XJOOJW4GiR^2u4&WV zIC$hTDOc<}Q;mt0(aRl8t!|{YE}@4&$12q>v!SWEH;{5%n%CtEZt#TX(xv z@?$M+E*M9WOb^_DI=O0}*#7`Dtc;Gq@JJXv`qV;{p({O$_K}kFB#TOt7Fy$Ve67p713rWQ0Iyvd&kdES5viV0_mG{TP6^39hHyP;`&qa9wV^MO zZp`aF9vcfiw|6s{Im}4dP}tA6sHzuN%^b|pq>`k7sFP_N5zrIQU(T=D)t2mqQZd-I z6gII(B=JM@kU;rCAbQtF;f*PED;=)`Ce_ARR3 ztJN-a9WPIV$@@BzM=g$PSkjC5QH9q1qRpps5N{^*%)7Xs-t!);4Ic}joXUrA0HZ~7+ z{m!sM|{F-#No)gW8c=8&{FefJSgvsKMmW^GOZ3V5I2|gZD;Y`` z>Q_MHy+s}Ca71Xsn$wue*|OTlxDv!K)}R`K%$r1OM=;es<@k(CoB~63HAqi_MmZeg zH4aSCL3TDGv@Y3+#yttMJhCOC7W+_wHvS(7khC+XY!y=J|$4J*jfbxb*CG>t$^zQnJdvpnJ zXylhhE61uk80;;V9LU)I^oF@ntR1;CJ^=Np>fQH`SD+p2+RZ#1V|c66EzNKsCV4sj|FkODQB|=Yvbxd5@X)I}Jy~Fxgv~i1~p7 zX~#U*n`+vWv+j#03;^0sr9Wj9Qj=$*O{)2exFRqkBj)Q~e`|X+*|v=xvViVCg&xfX z+0b8VaT|q;0a@EB!y{u%xx=rqw1%crD)hi)d;(uwf>a4EeK>AbWF`Vb{=mX0o*+e6;g z&^<*h#ByC-#}o=)M_{NuPC8Q9Yf{nFCcBO?G|c`|>_EpmN2O7C=C_(i-Ac&`!>{Ag zsZ_S)x@o-*n$N=e#rz@SiX~PV21!83{4++4s_V8EsFt?vG;hpk%aT2ZHEO*stj<_` zLe(N`F9z!xVYx4-i?p2r?T%F?NY6au*028n!eQbFUNn6tbGsB(3UXLyE7z#O$o8Sm zi`|l;iRg@H@UFM^cip8svQIHG@>O+o)(OaS>eY-I4+H<06_-z2eAf zPf0|A*T&1Z3GK?L`&iv0-c0W>1Z-9Eh0%){8j zvFWgQ_S)tjE=%X@BpUE}*p`BiqcxE*67@f9~^B#ZVIam@&rwiuJe8hf-T5yl8T zDx+hbt@*SmTr&%XI3Q;L`qt7z11oenIRN&d#^O~H(#1;)f3!x#u33Lt(~d~t0aO9> zp+zA%x3N{ATUdO_I`$PNqzu;MjHgpW(P&g#Z&N1b+ByN(6^my| zNS&^NlD~UB4M82V$jA3&0oxrZG*!|sLcOC>sQG2Zi6|JOqw%08`0Q}*1 zjP%DJms;d4wP#=5$lyfma~V=L0yrS_4N5o*ec=kq{CZ`DNqKLpqaQL&BHz>ogk$i*r)#TU`ul|2 zPeuKG<+(4T^05SgV_3s8X9co2sBARBaT~SNsWy7-0}buRYUPzE#rI=#y49T@hhUEK z2*j9p*?wKeB#M58Xv=vUL|7_ugU}Ac+PW#mN-s+obH&YA-TuhZq}v<90-??-OUO#^ zeB!i>REnKP4w@I|zPGi364^qk@qt`^t0dA{%M@}2b^xg-p$8-RR*H&_wz-nTI+{6h znwR%xVqMMwHG@6mh8W9`aslAh(Y3TVWTfSJ5VoYpdjQA-5CQH1>siyZCiY0zbd2M6 z@GvPxP3UhrYvNY1B$vKvD#sxD)?}NJ$n>GyB9P0Q7PZrEE}214%YZxOg)QaV4Ju`S zxt0E6lH9asN4jDe#s^TNwN|#eX(TZz3b^We)~ZS9Cl<9fH2qF{H$F_bD%|Isk3;QM z5n~Uu5y1Q2^z4auIvK3t)!>%+K(R{C^F$kV-~i+wU*}fsCbN4vt?ikjRg7#n&(Dv% zKOWu1X8ejxn9xOQ7a%bJ9tq&_ilkQZ2mSOiC<7;qf&D1Cdz-L%a#=J{kPh4p#~}Sh zP>Sp#sgxB~?~b$z#8Bz)5maUX1Cla0;+9!M$WA~5sn1_}cMA+UeD>Dv$kE87k%Qa5 zYUQQOZIvs#JP=P>E*3IkxkDo5f)VO?tvjC&hm{)I5IGrEYz$(P)Ds(TsPAzay1yeQ zZZY($R{AWC;bkH@0QmxAjyTO$gf`08QAU?D##_*?KhM28R*q%31lvCHfD(Y5k@TTl zE{LvwR^5V3t z{28ySs~c2C1byj11M#JG+zeZp-U#E2usK{Y9=+?LyYLnA@0Q~J&fX$(^S4TBOSl-e z@-VdC+myCWN&Z!CO&a1D580pXfqKk}b^+)+P}5+VC|+EW$;rqZX02QJUsb<_=kmmG zpgCvbI2{1$Y1-N`T{9lyee&*R3!YAEOn7c#5Hm4~SukT);F0h3q3$%*hH3XUk=rx2 zQeV2nw+B6iUQHs_!w&7bU;zU>1B#YWR_0x-y}qY`RWEJ^^S7RUv{)`N@W=vFa+5zJ6iz8Z3r@%y(d}?WY|oCXgt}By-91p|G*rrh_uyz39BQ zcXisyA_BPjXWEndKlo%adgQUCZzC>eYh1S-g_}Rw?kyB4US_v%m-5bY_*7rm<;LKk z1I~HootU|oa@u%*Mn(m0r3~D2CNua|1E9C?yJzvGZy@IW@y%LY$8Qd&Y4dKlbrIlj z0y!r?g>Ih+FC{XpdEK0!*0FS(cRCe#+QtkQ5!&k(mlCm9;+0vO9D!SMcrm1JnpqEg z0tfV@=}D_Hm3Sv~%sH-MxYcj26^V{kEQIwVaK&Gj!Wxaa1_kqn@eGAL9Mn2ZMMlPv zuO4gMyW#y(8*M~Qdov_4$HBoo?idvXu7#*cDS}Ntl^RJJ^o?ERgc4W7j~OXg~!Q+!!n^GKVQzX`nBZrc4(!D zijA&@$En-PXQ(_aBa6APn^>;iGxH4d`qT@c{f=^rrb@CaV=PPm04~3UTA>uvw#2+c zYS%HI`C1#2g_I(w$Up50?5+GIES7JMktPaKRm`s9kc)3Fcf|q>r=|5(9#JR*^`}&V7ux(eHxw zJu_Vz-Y@YjXl2gMK1(~~Y;Iv)drt#(LHSW(`$LL*eMAHEJ^-%Se$K%BtIxG7589r5 z{{U+MZfED(yP0h58y_n0eJHTiRC()}3BxhXb~9*}va>RP2|1-={i-;oQhJ=%XC{Ge zbA{_iD<8E-5XoH}-W~b=T!Bi=wKQ{yFO=J8)NPv5{h(vc0U4~@vmTF9!rdHos)#(s zRfsWj!BNT2wJv>ZPbs4n#!n`iH0$W&ibHPYWnIg(XY=%{yA_eC6v-reACPM;ZalHg z9Kz)^&YNh%Dx+{Y2Q^MR?4P|(=vI+q>`tVyWdQZXLve8XavO@4(MdA4p%ift%DuaE zHB#b9CQ}+o$vp_H+~R2z%<(IUOo*WK+*IZV3x-m$hTKB;agXE4bKU`gz8f`2-fP4PYEl*9cR*uVFsjD|msYULR+m+Tdw8y6R{J`{2;mn$)}N%vzw0C4wDjCY zYAnW@Z&Py880_-d+DNKqb(cAO5!$Pjy~*Z+_(*m8o5i(RY}l8L_Z0s4?kYV(>SL)v+(0Yp_+Ff{pHweq=rJ=uqljidEM*uqm>Jpl|Qkjcu`=P z6<0VcNId$A)J7a}$nQX|%2s+c{1WebyK)G_aCsh>tyo(f@^jY|?#Uq641+(p5O;r6(pcAb3*bDumUDDfd~T z1mn|!D{Q+-mE1z(s63z2moovncAEvHfu+0in4k>1KvG9>=|omm(Xo6xH@lNk$8%Ii z85|M^u&R+>z-9Y7%^#Qt`n}wWn=#zDAYup%2=qKtR&JiuhC=#1w2$UVXA{5ow{!Z` ztA7Jav2G(SKY%#T#&qdIm#YwR>l8a({e_CIBc`QJz|!$tDW1 z!voDZ0As)h``PN$&T+OlNE6H(i8vWK82*(C+H59EYmKT;%L@)iuRr5RZp^$?Ghj~0 z4^p_v{KZQGZAC`fa!v^2pK78nVNxuh3nRH5S&06XI@_3$agd<*<1|=T1&x6KwnjS& z73LBXlqv@q<38D-2ZdFBe>$u7$vKhjW&m^ugyxW0mx3-fs0EKtddibuhA6HrA=vlO zjow6=7(KGx1suqw2{T4{zRMYsXy0h$kT@03Yd#~JPlhQaw~kN%S7eukBy=Yn8kaPS zjVUvF9ZJS`D;>h9$jSbd=Q^gdX>n~H$UK!lCes?W!P%Ai=RUO#p{m2&?Z_U{#3oBfzhUp;k;j>&vm~KEUUt}1adlyElbe(@x{{RZZ zp7vshLj+lNG7;2*dC#X5bF#ghUqfbZ5WFbGm0YTpAQBD*b8}9L<2o>Xa@u;0WDVzkV? z>D;bDZAKVYZ?`fo<^c&j@GBDk08zY2nkb@IjeyB-@T{cQ%<7{RZ+l|Ime${84+#)D zHt`@FipiT)y0}j+M+%Miml!6YNn2w{DBnY_wDAmgkwY7@q`eu31M#jt0|ArwyMR0l z98`1N=*{T3+RiDnq}T2A^|$jD;wao?;ZXkY{b`z3gJ-Eh`^&Q`EQA6a@$&Kb=CpD# zaBf;7lNT(qfDQ#ss_3`cSPyX+5xWqq4u3j_aU?oh=GaqkVmpJyWrA`J(}B|!H)G^R zf=XIND#T=rwlPe$AhL|$3Z(_qhU{xC%(k*0HW4xt)DG1vnB%oBlI5|$Zj_xEjZ_nq zQ*~s%X4X5SkOsi|RV#a0ILV1f{&Z@#z7oSNy!R+c{{WUjnNQ+HQc;&iL!9IGqB%8R3r@$$v}JS9 zc|=w%jjCz)uQWF1@iB!H{J772dex{|^)q#wTHEeB8%v1(&6p$d*RUs%SoYeq^3Ea_ z=@{-7w@BoZW|6otLo2H~Tz|_p01^1p?_rH%VKnIz?B1w)wJCDF8#SjI^rP(~8!qjC znO4Wl)x6iZkU;1QFeGJi&E2x71fI22zi0Dm)FY)^G``-D==$M}Mi z`qgq1bJ0_8mJ1e*zqUR-LvajYneC-qWZSvH{OLG_%{ldJJ0%Tq zBgQjpYNTE;(ah#4BG~Kl?`6m8DLBip(_PMq zF1HQ>5PPp5rBjan(%}O|Ai?NC&1BJOllj|8Urv=qc`(6)1KNuP%Tio>^XpU|UNg@$ z>=z`OY=hpY%QiAInrQ{XX%B8iQA zcOOGl#F>#5%Joo>Is> zCUy?O@L}`<<4h zJ+Oe>Oy<>hZImDdUpyVkdvn*0mFIU_y~W1EbnS3JATjI@^Qe_XYZ@(eCe$sVRz-sL zDdTP!AS!o#4L03vSc0RT)i*RtLd2Jl z3GQaM1&5We5^>lFaQTA2PU&`WzESMBY{~;O6JjWjBJSKC8jNcGI-5M3{n6m zBzNMLjGMlq#2QGOayu%6$Qc5-|U}%O4c`Rs%i%I+Apyk zgiVIXVe5)~>1fV5D|{~HJwHpgYsn+McY#$%BoUQv*&dZG+QsJyM7*-$i}V<*{j_E9 zl})((YUFygul9PP3z%+HkCo$Zy`NFr6$AeOq#x+RI{^9w5T0;^EEQPrfTT02C2S)^h{^_oZPSUh_8H%#G1Cp(~^X8@5 zZi`FI^a(}Dt`{t`QrPK;VA8I>(suG_-;#N*&ZD8vE7ahWXq>KQ!7Nks`^!^ zE8g$#Gj&>4nV1RV2Px|QnMSV?1b8EcjyhoaQ! zZFDK7o{r=?du^~jQszPTfbMG@E+t7k$l0Sm?9t>j?YcTAR=TrQe5(VHrn2zQY;0TG81cvzWkqdsI~Vl%78yU)2^jtrRTt|(7cO;~;Gb+3GL{|i zaZw^ZeWG=Q*@vGt~x z5q;GRM^V;*EhB$;52+QICchjlw~jpxJ24xmxEVMCvTe1yX{Ax-1|;KXI5fFfT-TEN z3s}p)AXIlY7WdFNONNNL;{^2l4Njdv&$;n*E43kvxNf=NkIu8F@VYeKeA=EPBb}*& zf6vmiN&?i~H}?1O#?ZaIv9kQu@{BqW=~ON4#k3oAt9Fo){mVM|Rqyw#y0Q5Fq{I?&y6DX~UF8WnH=I1st~pJ9HVL-(6f=Di$R{!6XnH9r>q+ zk(g`*k=P)NDxDbJ@DnCFof)00PM1dxKPNtyg0h)l4mUx3__4>+u4B+iIN zOQ{?a#F4B@QBei~ErUV^z#+0bU>eNWvMwS20G_e_Em4ML>CfV55h1vTAL}Fk0KTd+ z#IMXSe-l6t7ZB$m)BgLWDJ0-F2e32&C)!l-91;Hi0RI45kA2PgH3HBK$qA;i-Y)d4LRCy>0-=BatK?i{4<;MHc6G_F;LGgKw% z6p>Q%K6Xc?Hs(eWZ2)wv<+$v~{LG9RK@((Wy-uN_ziX@8c`(B*xRYV;{69ljhcOX` z>sNM9p}BpliO5TMRby5R3mo8z$4MiRu*_5*qzY1Umr*5SywK;eys?t%?sbk&x??!Q zAAI`P4l=AnZdn)(2lb;&<9iL-rpk18%DvtMa!8KaZ^@mgyAY;Pogb zt2L+Y;sa^rSx9WJxW+1LYgLwQgyB>if!4H7P0aZs-jOmSib*GDRZ+a>YOV+6T_hH; z51FD?9+=PLRk5azbDWFLhE3Uc&rQjYYqJSuHdbqOMU#!_fG|G8p!7U+sB>$fqNfMc zz|(clu`v=`{h)5n=^U0`J9e$B?JrKdxD2-8BHN9rD=E$~o^zjEcci4PsvD-)!yzH@ zRf#A48f9bCHD2KkHG3U0YLQL%ay_e?vbVlQDmJ!y zklg_9TE)cXojLV4FK3c<4;zJOl=4q+O3Pb$Zs#eND;UcW+J@EiM8+-YaRuvMx<;`d zGaoOptn2HBYpEoSHm}{tmn70DRAP5V0yvc2^12S~^Z}x|&mM3qs%)wQ-U~%};HMxy8m&SU5 zPgF~3y$C0n=8&_wc0EZwX&pAV3sB7!t)Z)Dv24~=k8=}(aapilO$ZAcW1gL<-$1pV z#^tmT#i+pQfkP{9KZR%7UQF7G+)0cDJPw^JNVMgl4QOp7L24f}M(c z3f{(M{8uqB%7fRqtV>xPoFcLJIj3P8vfc=r9$-P+HO<}J%PZzWeqI9lQm`Eg-8{Ba zPK1_Fa5wh)Ry1C7-NnODx4~0Yyp$lQv;;Y?k-wImv@?@ zf-{moI-vsnmDkOX0KBp*a%gCh{;Q_R6t6A3UStf|9$r|X?g*=J>gr?Mtno4R!Ozx` zx*mokFx*PaBDZo>WbumKZFVRMB!tT#AAOVzQ@a(BOHQ@arg0-e%DBhxM^W^rUigOK zpeVRj`={26FwoeJ2<>6o(rw4%isPo#C!ZzdKTOl*1GI-!w1XL#vwwM-xM=Pf95zYC z9HKogElwHNYdBziD}}Vx=2L>oBNzwPrNC!lKA8iO=`mr~X{<;!PB_jxoKmv`NuUuP zFmY9W*}QzG^`lTM`GkKL(-lpmC$Bm7pk=dh$KEH~6-dj1NFzU5EHYM`YyLddS7zm# zJ4OdeKv;RZpU$cocpd$-(tsqAY#}(@ahzurUEkHX^rkUnl=bR8Jt`JB$Q?Z>0a^D9 QW2aM^g^A~%(vZ*p*?7fNG5`Po literal 0 HcmV?d00001 diff --git a/modules/lora/networks.py b/modules/lora/networks.py index 9103fee37..2552922fa 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -198,7 +198,7 @@ def add_network(filename): except OSError as e: # should catch FileNotFoundError and PermissionError etc. shared.log.error(f'LoRA: filename="{filename}" {e}') - candidates = list(files_cache.list_files(shared.cmd_opts.lora_dir, ext_filter=[".pt", ".ckpt", ".safetensors"])) + candidates = sorted(files_cache.list_files(shared.cmd_opts.lora_dir, ext_filter=[".pt", ".ckpt", ".safetensors"])) with concurrent.futures.ThreadPoolExecutor(max_workers=shared.max_workers) as executor: for fn in candidates: executor.submit(add_network, fn) diff --git a/modules/shared.py b/modules/shared.py index a4afc5b59..e866c4539 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -477,7 +477,7 @@ def get_default_modes(): "sd_model_checkpoint": OptionInfo(default_checkpoint, "Base model", DropdownEditable, lambda: {"choices": list_checkpoint_titles()}, refresh=refresh_checkpoints), "sd_model_refiner": OptionInfo('None', "Refiner model", gr.Dropdown, lambda: {"choices": ['None'] + list_checkpoint_titles()}, refresh=refresh_checkpoints), "sd_unet": OptionInfo("None", "UNET model", gr.Dropdown, lambda: {"choices": shared_items.sd_unet_items()}, refresh=shared_items.refresh_unet_list), - "latent_history": OptionInfo(16, "Latent history size", gr.Slider, {"minimum": 1, "maximum": 100, "step": 1}), + "latent_history": OptionInfo(16, "Latent history size", gr.Slider, {"minimum": 0, "maximum": 100, "step": 1}), "offload_sep": OptionInfo("

Model Offloading

", "", gr.HTML), "diffusers_move_base": OptionInfo(False, "Move base model to CPU when using refiner", gr.Checkbox, {"visible": False }), diff --git a/modules/ui_extra_networks.py b/modules/ui_extra_networks.py index 6389da194..bc5afc8d2 100644 --- a/modules/ui_extra_networks.py +++ b/modules/ui_extra_networks.py @@ -392,6 +392,7 @@ def update_all_previews(self, items): debug(f'EN mapped-preview: {item["name"]}={found}') if item.get('preview', None) is None: item['preview'] = self.link_preview('html/card-no-preview.png') + print('HERE', base) debug(f'EN missing-preview: {item["name"]}') self.preview_time += time.time() - t0 @@ -464,13 +465,15 @@ def register_pages(): from modules.ui_extra_networks_checkpoints import ExtraNetworksPageCheckpoints from modules.ui_extra_networks_vae import ExtraNetworksPageVAEs from modules.ui_extra_networks_styles import ExtraNetworksPageStyles - from modules.ui_extra_networks_history import ExtraNetworksPageHistory - from modules.ui_extra_networks_textual_inversion import ExtraNetworksPageTextualInversion register_page(ExtraNetworksPageCheckpoints()) register_page(ExtraNetworksPageVAEs()) register_page(ExtraNetworksPageStyles()) - register_page(ExtraNetworksPageHistory()) - register_page(ExtraNetworksPageTextualInversion()) + if shared.opts.latent_history > 0: + from modules.ui_extra_networks_history import ExtraNetworksPageHistory + register_page(ExtraNetworksPageHistory()) + if shared.opts.diffusers_enable_embed: + from modules.ui_extra_networks_textual_inversion import ExtraNetworksPageTextualInversion + register_page(ExtraNetworksPageTextualInversion()) if shared.native: from modules.ui_extra_networks_lora import ExtraNetworksPageLora register_page(ExtraNetworksPageLora()) From 8dc5cb940cdb09cad6fc83255cec1adbb15005b3 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 27 Dec 2024 09:06:55 -0500 Subject: [PATCH 160/249] ltxvideo optimizations Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 3 ++- modules/ui_extra_networks.py | 1 - scripts/hunyuanvideo.py | 4 ++-- scripts/ltxvideo.py | 23 ++++++++++++++++++++--- 4 files changed, 24 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c7f70393..481223a0d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,11 +4,12 @@ ### Post release -- Add legacy option to use old LoRA loader in *settings -> networks* - Add granular VAE tiling options in *settings -> variable auto encoder* +- Add legacy option to use old LoRA loader in *settings -> networks* - Add sigma calculation to VAE preview, thanks @Disty0 - Fix live preview image sizes in modern and standard UI - HunyuanVideo optimizations: full offload, quantization and tiling support +- LTXVideo optimizations: full offload, quantization and tiling support - Do not show disabled networks ## Update for 2024-12-24 diff --git a/modules/ui_extra_networks.py b/modules/ui_extra_networks.py index bc5afc8d2..146fb8fbc 100644 --- a/modules/ui_extra_networks.py +++ b/modules/ui_extra_networks.py @@ -392,7 +392,6 @@ def update_all_previews(self, items): debug(f'EN mapped-preview: {item["name"]}={found}') if item.get('preview', None) is None: item['preview'] = self.link_preview('html/card-no-preview.png') - print('HERE', base) debug(f'EN missing-preview: {item["name"]}') self.preview_time += time.time() - t0 diff --git a/scripts/hunyuanvideo.py b/scripts/hunyuanvideo.py index f61c00fdb..61ef07f30 100644 --- a/scripts/hunyuanvideo.py +++ b/scripts/hunyuanvideo.py @@ -38,7 +38,7 @@ def hijack_decode(*args, **kwargs): res = shared.sd_model.vae.orig_decode(*args, **kwargs) t1 = time.time() timer.process.add('vae', t1-t0) - shared.log.debug(f'Video: decoder={vae.__class__.__name__} tile={vae.tile_sample_min_width}:{vae.tile_sample_min_height}:{vae.tile_sample_min_num_frames} stride={vae.tile_sample_stride_width}:{vae.tile_sample_stride_height}:{vae.tile_sample_stride_num_frames} time={t1-t0:.2f}') + shared.log.debug(f'Video: vae={vae.__class__.__name__} tile={vae.tile_sample_min_width}:{vae.tile_sample_min_height}:{vae.tile_sample_min_num_frames} stride={vae.tile_sample_stride_width}:{vae.tile_sample_stride_height}:{vae.tile_sample_stride_num_frames} time={t1-t0:.2f}') return res @@ -47,7 +47,7 @@ def hijack_encode_prompt(*args, **kwargs): res = shared.sd_model.vae.orig_encode_prompt(*args, **kwargs) t1 = time.time() timer.process.add('te', t1-t0) - shared.log.debug(f'Video: encode cls={shared.sd_model.text_encoder.__class__.__name__} time={t1-t0:.2f}') + shared.log.debug(f'Video: te={shared.sd_model.text_encoder.__class__.__name__} time={t1-t0:.2f}') shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) return res diff --git a/scripts/ltxvideo.py b/scripts/ltxvideo.py index 007c4f4cc..ab2e3ed95 100644 --- a/scripts/ltxvideo.py +++ b/scripts/ltxvideo.py @@ -4,7 +4,7 @@ import gradio as gr import diffusers import transformers -from modules import scripts, processing, shared, images, devices, sd_models, sd_checkpoint, model_quant +from modules import scripts, processing, shared, images, devices, sd_models, sd_checkpoint, model_quant, timer repos = { @@ -32,9 +32,24 @@ def load_quants(kwargs, repo_id): def hijack_decode(*args, **kwargs): - shared.log.debug('Video: decode') + t0 = time.time() + # vae: diffusers.AutoencoderKLHunyuanVideo = shared.sd_model.vae + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model, exclude=['vae']) + res = shared.sd_model.vae.orig_decode(*args, **kwargs) + t1 = time.time() + timer.process.add('vae', t1-t0) + shared.log.debug(f'Video: vae={shared.sd_model.vae.__class__.__name__} time={t1-t0:.2f}') + return res + + +def hijack_encode_prompt(*args, **kwargs): + t0 = time.time() + res = shared.sd_model.vae.orig_encode_prompt(*args, **kwargs) + t1 = time.time() + timer.process.add('te', t1-t0) + shared.log.debug(f'Video: te={shared.sd_model.text_encoder.__class__.__name__} time={t1-t0:.2f}') shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) - return shared.sd_model.vae.orig_decode(*args, **kwargs) + return res class Script(scripts.Script): @@ -128,7 +143,9 @@ def run(self, p: processing.StableDiffusionProcessing, model, model_custom, deco ) sd_models.set_diffuser_options(shared.sd_model) shared.sd_model.vae.orig_decode = shared.sd_model.vae.decode + shared.sd_model.vae.orig_encode_prompt = shared.sd_model.encode_prompt shared.sd_model.vae.decode = hijack_decode + shared.sd_model.encode_prompt = hijack_encode_prompt shared.sd_model.sd_checkpoint_info = sd_checkpoint.CheckpointInfo(repo_id) shared.sd_model.sd_model_hash = None shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) From 16f4101fe44fd1d4b1cb202d7ebdb6309d46ab5b Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 27 Dec 2024 09:44:19 -0500 Subject: [PATCH 161/249] minor xyz patch Signed-off-by: Vladimir Mandic --- scripts/xyz_grid_draw.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/scripts/xyz_grid_draw.py b/scripts/xyz_grid_draw.py index 23e98e0fe..29b083d5f 100644 --- a/scripts/xyz_grid_draw.py +++ b/scripts/xyz_grid_draw.py @@ -97,10 +97,10 @@ def index(ix, iy, iz): process_cell(x, y, z, ix, iy, iz) if not processed_result: - shared.log.error("XYZ grid: Failed to initialize processing") + shared.log.error("XYZ grid: failed to initialize processing") return processing.Processed(p, []) elif not any(processed_result.images): - shared.log.error("XYZ grid: Failed to return processed image") + shared.log.error("XYZ grid: failed to return processed image") return processing.Processed(p, []) t1 = time.time() @@ -109,7 +109,10 @@ def index(ix, iy, iz): idx0 = (i * len(xs) * len(ys)) + i # starting index of images in subgrid idx1 = (len(xs) * len(ys)) + idx0 # ending index of images in subgrid to_process = processed_result.images[idx0:idx1] - w, h = max(i.width for i in to_process), max(i.height for i in to_process) + w, h = max(i.width for i in to_process), max(i.height for i in to_process if i is not None) + if w is None or h is None or w == 0 or h == 0: + shared.log.error("XYZ grid: failed get valid image") + continue if (not no_grid or include_sub_grids) and images.check_grid_size(to_process): grid = images.image_grid(to_process, rows=len(ys)) if draw_legend: From fabdcf25e9b7b3fb3dec5c775b42235ddd63cd20 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 27 Dec 2024 14:55:16 -0500 Subject: [PATCH 162/249] css optimizations Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 1 + javascript/extraNetworks.js | 3 ++- javascript/gallery.js | 1 + javascript/logMonitor.js | 9 ++++++++- javascript/sdnext.css | 2 +- 5 files changed, 13 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 481223a0d..3367e98dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ - HunyuanVideo optimizations: full offload, quantization and tiling support - LTXVideo optimizations: full offload, quantization and tiling support - Do not show disabled networks +- CSS optimizations when log view is disabled ## Update for 2024-12-24 diff --git a/javascript/extraNetworks.js b/javascript/extraNetworks.js index 1d1bcfb24..c42ba5761 100644 --- a/javascript/extraNetworks.js +++ b/javascript/extraNetworks.js @@ -440,7 +440,8 @@ function setupExtraNetworksForTab(tabname) { for (const el of Array.from(gradioApp().getElementById(`${tabname}_extra_tabs`).querySelectorAll('.extra-networks-page'))) { const h = Math.trunc(entry.contentRect.height); if (h <= 0) return; - if (window.opts.extra_networks_card_cover === 'sidebar' && window.opts.theme_type === 'Standard') el.style.height = `max(55vh, ${h - 90}px)`; + const vh = opts.logmonitor_show ? '55vh' : '68vh'; + if (window.opts.extra_networks_card_cover === 'sidebar' && window.opts.theme_type === 'Standard') el.style.height = `max(${vh}, ${h - 90}px)`; // log(`${tabname} height: ${entry.target.id}=${h} ${el.id}=${el.clientHeight}`); } } diff --git a/javascript/gallery.js b/javascript/gallery.js index 05e594e4c..32f261e12 100644 --- a/javascript/gallery.js +++ b/javascript/gallery.js @@ -396,6 +396,7 @@ async function initGallery() { // triggered on gradio change to monitor when ui el.search = gradioApp().querySelector('#tab-gallery-search textarea'); el.search.addEventListener('input', gallerySearch); el.btnSend = gradioApp().getElementById('tab-gallery-send-image'); + document.getElementById('tab-gallery-files').style.height = opts.logmonitor_show ? '75vh' : '85vh'; const intersectionObserver = new IntersectionObserver((entries) => { if (entries[0].intersectionRatio <= 0) galleryHidden(); diff --git a/javascript/logMonitor.js b/javascript/logMonitor.js index 9b915e6da..151bb9df5 100644 --- a/javascript/logMonitor.js +++ b/javascript/logMonitor.js @@ -44,10 +44,17 @@ async function logMonitor() { if (modenUIBtn) modenUIBtn.setAttribute('error-count', logErrors > 0 ? logErrors : ''); }; + document.getElementById('txt2img_gallery').style.height = opts.logmonitor_show ? '50vh' : '55vh'; + document.getElementById('img2img_gallery').style.height = opts.logmonitor_show ? '50vh' : '55vh'; + + if (!opts.logmonitor_show) { + Array.from(document.getElementsByClassName('log-monitor')).forEach((el) => el.style.display = 'none'); + return; + } + if (logMonitorStatus) setTimeout(logMonitor, opts.logmonitor_refresh_period); else setTimeout(logMonitor, 10 * 1000); // on failure try to reconnect every 10sec - if (!opts.logmonitor_show) return; logMonitorStatus = false; if (!logMonitorEl) { logMonitorEl = document.getElementById('logMonitorData'); diff --git a/javascript/sdnext.css b/javascript/sdnext.css index 69d67baf8..2e85c6988 100644 --- a/javascript/sdnext.css +++ b/javascript/sdnext.css @@ -324,7 +324,7 @@ table.settings-value-table td { padding: 0.4em; border: 1px solid #ccc; max-widt div:has(>#tab-gallery-folders) { flex-grow: 0 !important; background-color: var(--input-background-fill); min-width: max-content !important; } .gallery-separator { background-color: var(--input-background-fill); font-size: larger; padding: 0.5em; display: block !important; } #html_log_gallery { font-size: 0.95em; } -#gallery_gallery { height: 60vh; } +#gallery_gallery { height: 63vh; } #gallery_gallery .thumbnails { display: none; } #gallery_gallery .preview { background: none; } #gallery_gallery img { object-fit: contain; height: 100% !important; } From dbdb929accf62847a5c5dde8991eb377be95adbc Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 27 Dec 2024 16:33:19 -0500 Subject: [PATCH 163/249] sampler flow shift options and fix img2img Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 2 ++ modules/images_grid.py | 5 +++-- modules/processing_args.py | 5 ++++- modules/processing_diffusers.py | 8 +++++--- modules/sd_samplers_diffusers.py | 11 ++--------- modules/shared.py | 4 ++-- modules/ui_sections.py | 16 +++++++++++++--- scripts/xyz_grid_draw.py | 2 +- 8 files changed, 32 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3367e98dc..d8bd7ff75 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,10 +8,12 @@ - Add legacy option to use old LoRA loader in *settings -> networks* - Add sigma calculation to VAE preview, thanks @Disty0 - Fix live preview image sizes in modern and standard UI +- Fix image width/height calculation when doing img2img - HunyuanVideo optimizations: full offload, quantization and tiling support - LTXVideo optimizations: full offload, quantization and tiling support - Do not show disabled networks - CSS optimizations when log view is disabled +- Sampler options: add flow shift and separate dynamic thresholding from dynamic shifting ## Update for 2024-12-24 diff --git a/modules/images_grid.py b/modules/images_grid.py index 51371a0fe..a7f10396e 100644 --- a/modules/images_grid.py +++ b/modules/images_grid.py @@ -42,10 +42,11 @@ def image_grid(imgs, batch_size=1, rows=None): imgs = [i for i in imgs if i is not None] if imgs is not None else [] if len(imgs) == 0: return None - w, h = max(i.width for i in imgs), max(i.height for i in imgs) + w, h = max(i.width for i in imgs if i is not None), max(i.height for i in imgs if i is not None) grid = Image.new('RGB', size=(params.cols * w, params.rows * h), color=shared.opts.grid_background) for i, img in enumerate(params.imgs): - grid.paste(img, box=(i % params.cols * w, i // params.cols * h)) + if img is not None: + grid.paste(img, box=(i % params.cols * w, i // params.cols * h)) return grid diff --git a/modules/processing_args.py b/modules/processing_args.py index e0356dcff..c744c3d14 100644 --- a/modules/processing_args.py +++ b/modules/processing_args.py @@ -22,7 +22,7 @@ def task_specific_kwargs(p, model): if len(getattr(p, 'init_images', [])) > 0: if isinstance(p.init_images[0], str): p.init_images = [helpers.decode_base64_to_image(i, quiet=True) for i in p.init_images] - p.init_images = [i.convert('RGB') if i.mode != 'RGB' else i for i in p.init_images] + p.init_images = [i.convert('RGB') if i.mode != 'RGB' else i for i in p.init_images if i is not None] if (sd_models.get_diffusers_task(model) == sd_models.DiffusersTaskType.TEXT_2_IMAGE or len(getattr(p, 'init_images', [])) == 0) and not is_img2img_model: p.ops.append('txt2img') if hasattr(p, 'width') and hasattr(p, 'height'): @@ -262,6 +262,9 @@ def set_pipeline_args(p, model, prompts: list, negative_prompts: list, prompts_2 elif 'callback' in possible: args['callback'] = diffusers_callback_legacy + if 'image' in kwargs and len(getattr(p, 'init_images', [])) == 0: + p.init_images = kwargs['image'] if isinstance(kwargs['image'], list) else [kwargs['image']] + # handle remaining args for arg in kwargs: if arg in possible: # add kwargs diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 1c2128958..a108a1095 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -201,7 +201,8 @@ def process_hires(p: processing.StableDiffusionProcessing, output): if p.is_control and hasattr(p, 'task_args') and p.task_args.get('image', None) is not None: if hasattr(shared.sd_model, "vae") and output.images is not None and len(output.images) > 0: output.images = processing_vae.vae_decode(latents=output.images, model=shared.sd_model, full_quality=p.full_quality, output_type='pil', width=p.hr_upscale_to_x, height=p.hr_upscale_to_y) # controlnet cannnot deal with latent input - p.task_args['image'] = output.images # replace so hires uses new output + p.init_images = output.images # replace so hires uses new output + # p.task_args['image'] = output.images # replace so hires uses new output update_sampler(p, shared.sd_model, second_pass=True) orig_denoise = p.denoising_strength p.denoising_strength = strength @@ -290,8 +291,9 @@ def process_refine(p: processing.StableDiffusionProcessing, output): image = processing_vae.vae_decode(latents=image, model=shared.sd_model, full_quality=p.full_quality, output_type='pil', width=p.width, height=p.height) p.extra_generation_params['Noise level'] = noise_level output_type = 'np' - if hasattr(p, 'task_args') and p.task_args.get('image', None) is not None and output is not None: # replace input with output so it can be used by hires/refine - p.task_args['image'] = image + if p.task_args.get('image', None) is not None and output is not None: # replace input with output so it can be used by hires/refine + # p.task_args['image'] = image + p.init_images = [image] shared.log.info(f'Refiner: class={shared.sd_refiner.__class__.__name__}') update_sampler(p, shared.sd_refiner, second_pass=True) refiner_args = set_pipeline_args( diff --git a/modules/sd_samplers_diffusers.py b/modules/sd_samplers_diffusers.py index 6c05b2045..13a5c3ad9 100644 --- a/modules/sd_samplers_diffusers.py +++ b/modules/sd_samplers_diffusers.py @@ -235,16 +235,9 @@ def __init__(self, name, constructor, model, **kwargs): if 'beta_end' in self.config and shared.opts.schedulers_beta_end > 0: self.config['beta_end'] = shared.opts.schedulers_beta_end if 'shift' in self.config: - if shared.opts.schedulers_shift == 0: - if 'StableDiffusion3' in model.__class__.__name__: - self.config['shift'] = 3 - if 'Flux' in model.__class__.__name__: - self.config['shift'] = 1 - else: - self.config['shift'] = shared.opts.schedulers_shift + self.config['shift'] = shared.opts.schedulers_shift if shared.opts.schedulers_shift > 0 else 3 if 'use_dynamic_shifting' in self.config: - if 'Flux' in model.__class__.__name__: - self.config['use_dynamic_shifting'] = shared.opts.schedulers_dynamic_shift + self.config['use_dynamic_shifting'] = True if shared.opts.schedulers_shift <= 0 else shared.opts.schedulers_dynamic_shift if 'use_beta_sigmas' in self.config and 'sigma_schedule' in self.config: self.config['use_beta_sigmas'] = 'StableDiffusion3' in model.__class__.__name__ if 'rescale_betas_zero_snr' in self.config: diff --git a/modules/shared.py b/modules/shared.py index e866c4539..9904309a3 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -822,8 +822,8 @@ def get_default_modes(): 'schedulers_beta_start': OptionInfo(0, "Beta start", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.00001, "visible": native}), 'schedulers_beta_end': OptionInfo(0, "Beta end", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.00001, "visible": native}), 'schedulers_timesteps_range': OptionInfo(1000, "Timesteps range", gr.Slider, {"minimum": 250, "maximum": 4000, "step": 1, "visible": native}), - 'schedulers_shift': OptionInfo(0, "Sampler shift", gr.Slider, {"minimum": 0.1, "maximum": 10, "step": 0.1, "visible": native}), - 'schedulers_dynamic_shift': OptionInfo(True, "Sampler dynamic shift", gr.Checkbox, {"visible": native}), + 'schedulers_shift': OptionInfo(3, "Sampler shift", gr.Slider, {"minimum": 0.1, "maximum": 10, "step": 0.1, "visible": False}), + 'schedulers_dynamic_shift': OptionInfo(True, "Sampler dynamic shift", gr.Checkbox, {"visible": False}), # managed from ui.py for backend original k-diffusion "always_batch_cond_uncond": OptionInfo(False, "Disable conditional batching", gr.Checkbox, {"visible": not native}), diff --git a/modules/ui_sections.py b/modules/ui_sections.py index fcf53cf70..839fa9025 100644 --- a/modules/ui_sections.py +++ b/modules/ui_sections.py @@ -217,7 +217,8 @@ def set_sampler_original_options(sampler_options, sampler_algo): shared.opts.save(shared.config_filename, silent=True) def set_sampler_options(sampler_options): - shared.opts.data['schedulers_use_thresholding'] = 'dynamic' in sampler_options + shared.opts.data['schedulers_dynamic_shift'] = 'dynamic' in sampler_options + shared.opts.data['schedulers_use_thresholding'] = 'thresholding' in sampler_options shared.opts.data['schedulers_use_loworder'] = 'low order' in sampler_options shared.opts.data['schedulers_rescale_betas'] = 'rescale' in sampler_options shared.log.debug(f'Sampler set options: {sampler_options}') @@ -253,6 +254,11 @@ def set_sampler_beta(sampler_beta): shared.opts.schedulers_beta_schedule = sampler_beta shared.opts.save(shared.config_filename, silent=True) + def set_sampler_shift(sampler_shift): + shared.log.debug(f'Sampler set options: shift={sampler_shift}') + shared.opts.schedulers_shift = sampler_shift + shared.opts.save(shared.config_filename, silent=True) + # 'linear', 'scaled_linear', 'squaredcos_cap_v2' def set_sampler_preset(preset): if preset == 'AYS SD15': @@ -286,10 +292,13 @@ def set_sampler_preset(preset): sampler_timesteps = gr.Textbox(label='Timesteps override', elem_id=f"{tabname}_sampler_timesteps", value=shared.opts.schedulers_timesteps) with gr.Row(elem_classes=['flex-break']): sampler_order = gr.Slider(minimum=0, maximum=5, step=1, label="Sampler order", value=shared.opts.schedulers_solver_order, elem_id=f"{tabname}_sampler_order") - options = ['low order', 'dynamic', 'rescale'] + sampler_shift = gr.Slider(minimum=0, maximum=10, step=0.1, label="Flow shift", value=shared.opts.schedulers_shift, elem_id=f"{tabname}_sampler_shift") + with gr.Row(elem_classes=['flex-break']): + options = ['low order', 'thresholding', 'dynamic', 'rescale'] values = [] values += ['low order'] if shared.opts.data.get('schedulers_use_loworder', True) else [] - values += ['dynamic'] if shared.opts.data.get('schedulers_use_thresholding', False) else [] + values += ['thresholding'] if shared.opts.data.get('schedulers_use_thresholding', False) else [] + values += ['dynamic'] if shared.opts.data.get('schedulers_dynamic_shift', False) else [] values += ['rescale'] if shared.opts.data.get('schedulers_rescale_betas', False) else [] sampler_options = gr.CheckboxGroup(label='Options', elem_id=f"{tabname}_sampler_options", choices=options, value=values, type='value') @@ -300,6 +309,7 @@ def set_sampler_preset(preset): sampler_beta.change(fn=set_sampler_beta, inputs=[sampler_beta], outputs=[]) sampler_prediction.change(fn=set_sampler_prediction, inputs=[sampler_prediction], outputs=[]) sampler_order.change(fn=set_sampler_order, inputs=[sampler_order], outputs=[]) + sampler_shift.change(fn=set_sampler_shift, inputs=[sampler_shift], outputs=[]) sampler_options.change(fn=set_sampler_options, inputs=[sampler_options], outputs=[]) diff --git a/scripts/xyz_grid_draw.py b/scripts/xyz_grid_draw.py index 29b083d5f..cd7eb8d1f 100644 --- a/scripts/xyz_grid_draw.py +++ b/scripts/xyz_grid_draw.py @@ -109,7 +109,7 @@ def index(ix, iy, iz): idx0 = (i * len(xs) * len(ys)) + i # starting index of images in subgrid idx1 = (len(xs) * len(ys)) + idx0 # ending index of images in subgrid to_process = processed_result.images[idx0:idx1] - w, h = max(i.width for i in to_process), max(i.height for i in to_process if i is not None) + w, h = max(i.width for i in to_process if i is not None), max(i.height for i in to_process if i is not None) if w is None or h is None or w == 0 or h == 0: shared.log.error("XYZ grid: failed get valid image") continue From a3ad3b6019b3d403c180027c3d876f14ff8073f3 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 27 Dec 2024 17:04:54 -0500 Subject: [PATCH 164/249] fix flux pipeline switches Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 19 ++++++++++--------- modules/sd_models.py | 25 +++++++++++++++---------- 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8bd7ff75..68d038d58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,16 +4,17 @@ ### Post release -- Add granular VAE tiling options in *settings -> variable auto encoder* -- Add legacy option to use old LoRA loader in *settings -> networks* -- Add sigma calculation to VAE preview, thanks @Disty0 -- Fix live preview image sizes in modern and standard UI -- Fix image width/height calculation when doing img2img -- HunyuanVideo optimizations: full offload, quantization and tiling support -- LTXVideo optimizations: full offload, quantization and tiling support -- Do not show disabled networks -- CSS optimizations when log view is disabled +- **HunyuanVideo** optimizations: full offload, quantization and tiling support +- **LTXVideo** optimizations: full offload, quantization and tiling support +- VAE tiling granular options in *settings -> variable auto encoder* +- LoRA loader add legacy option in *settings -> networks* +- Live preview: add sigma calculation, thanks @Disty0 +- UI: CSS optimizations when log view is disabled - Sampler options: add flow shift and separate dynamic thresholding from dynamic shifting +- Fix: do not show disabled networks +- Fix: live preview image sizes in modern and standard UI +- Fix: image width/height calculation when doing img2img +- Fix: Flux pipeline switches: txt/img/inpaint ## Update for 2024-12-24 diff --git a/modules/sd_models.py b/modules/sd_models.py index f44b8cb4c..f3d0f569f 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -1208,7 +1208,7 @@ def set_diffuser_pipe(pipe, new_pipe_type): 'StableVideoDiffusionPipeline', ] - n = getattr(pipe.__class__, '__name__', '') + has_errors = False if new_pipe_type == DiffusersTaskType.TEXT_2_IMAGE: clean_diffuser_pipe(pipe) @@ -1217,7 +1217,7 @@ def set_diffuser_pipe(pipe, new_pipe_type): # skip specific pipelines cls = pipe.__class__.__name__ - if n in exclude: + if cls in exclude: return pipe if 'Onnx' in cls: return pipe @@ -1225,9 +1225,9 @@ def set_diffuser_pipe(pipe, new_pipe_type): new_pipe = None # in some cases we want to reset the pipeline to parent as they dont have their own variants if new_pipe_type == DiffusersTaskType.IMAGE_2_IMAGE or new_pipe_type == DiffusersTaskType.INPAINTING: - if n == 'StableDiffusionPAGPipeline': + if cls == 'StableDiffusionPAGPipeline': pipe = switch_pipe(diffusers.StableDiffusionPipeline, pipe) - if n == 'StableDiffusionXLPAGPipeline': + if cls == 'StableDiffusionXLPAGPipeline': pipe = switch_pipe(diffusers.StableDiffusionXLPipeline, pipe) sd_checkpoint_info = getattr(pipe, "sd_checkpoint_info", None) @@ -1254,8 +1254,8 @@ def set_diffuser_pipe(pipe, new_pipe_type): return pipe except Exception as e: # pylint: disable=unused-variable shared.log.warning(f'Pipeline class change failed: type={new_pipe_type} pipeline={cls} {e}') - return pipe - else: + has_errors = True + if not hasattr(pipe, 'config') or has_errors: try: # maybe a wrapper pipeline so just change the class if new_pipe_type == DiffusersTaskType.TEXT_2_IMAGE: pipe.__class__ = diffusers.pipelines.auto_pipeline._get_task_class(diffusers.pipelines.auto_pipeline.AUTO_TEXT2IMAGE_PIPELINES_MAPPING, cls) # pylint: disable=protected-access @@ -1267,11 +1267,11 @@ def set_diffuser_pipe(pipe, new_pipe_type): pipe.__class__ = diffusers.pipelines.auto_pipeline._get_task_class(diffusers.pipelines.auto_pipeline.AUTO_INPAINT_PIPELINES_MAPPING, cls) # pylint: disable=protected-access new_pipe = pipe else: - shared.log.error(f'Pipeline class change failed: type={new_pipe_type} pipeline={cls}') + shared.log.error(f'Pipeline class set failed: type={new_pipe_type} pipeline={cls}') return pipe except Exception as e: # pylint: disable=unused-variable shared.log.warning(f'Pipeline class set failed: type={new_pipe_type} pipeline={cls} {e}') - return pipe + has_errors = True # if pipe.__class__ == new_pipe.__class__: # return pipe @@ -1282,8 +1282,13 @@ def set_diffuser_pipe(pipe, new_pipe_type): new_pipe.has_accelerate = has_accelerate new_pipe.current_attn_name = current_attn_name new_pipe.default_scheduler = default_scheduler - new_pipe.image_encoder = image_encoder - new_pipe.feature_extractor = feature_extractor + if image_encoder is not None: + new_pipe.image_encoder = image_encoder + if feature_extractor is not None: + new_pipe.feature_extractor = feature_extractor + if new_pipe.__class__.__name__ == 'FluxPipeline': + new_pipe.register_modules(image_encoder = image_encoder) + new_pipe.register_modules(feature_extractor = feature_extractor) new_pipe.is_sdxl = getattr(pipe, 'is_sdxl', False) # a1111 compatibility item new_pipe.is_sd2 = getattr(pipe, 'is_sd2', False) new_pipe.is_sd1 = getattr(pipe, 'is_sd1', True) From 2708056467838837f3e0925d70498abf056041e7 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 28 Dec 2024 08:14:37 -0500 Subject: [PATCH 165/249] fix lora legacy loader Signed-off-by: Vladimir Mandic --- extensions-builtin/Lora/networks.py | 1 + extensions-builtin/Lora/ui_extra_networks_lora.py | 2 ++ modules/ui_extra_networks.py | 2 +- modules/ui_javascript.py | 4 ---- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/extensions-builtin/Lora/networks.py b/extensions-builtin/Lora/networks.py index 1f02f3846..9f77aac1a 100644 --- a/extensions-builtin/Lora/networks.py +++ b/extensions-builtin/Lora/networks.py @@ -543,6 +543,7 @@ def network_MultiheadAttention_load_state_dict(self, *args, **kwargs): def list_available_networks(): + print('HERE') t0 = time.time() available_networks.clear() available_network_aliases.clear() diff --git a/extensions-builtin/Lora/ui_extra_networks_lora.py b/extensions-builtin/Lora/ui_extra_networks_lora.py index 4220b8e02..c5adbb81f 100644 --- a/extensions-builtin/Lora/ui_extra_networks_lora.py +++ b/extensions-builtin/Lora/ui_extra_networks_lora.py @@ -12,6 +12,7 @@ class ExtraNetworksPageLora(ui_extra_networks.ExtraNetworksPage): def __init__(self): super().__init__('Lora') self.list_time = 0 + shared.log.warning('Networks: type=lora method=legacy') def refresh(self): networks.list_available_networks() @@ -79,6 +80,7 @@ def find_version(): def create_item(self, name): l = networks.available_networks.get(name) if l is None: + print('HERE1') shared.log.warning(f'Networks: type=lora registered={len(list(networks.available_networks))} file="{name}" not registered') return None try: diff --git a/modules/ui_extra_networks.py b/modules/ui_extra_networks.py index 146fb8fbc..3594939b2 100644 --- a/modules/ui_extra_networks.py +++ b/modules/ui_extra_networks.py @@ -473,7 +473,7 @@ def register_pages(): if shared.opts.diffusers_enable_embed: from modules.ui_extra_networks_textual_inversion import ExtraNetworksPageTextualInversion register_page(ExtraNetworksPageTextualInversion()) - if shared.native: + if not shared.opts.lora_legacy: from modules.ui_extra_networks_lora import ExtraNetworksPageLora register_page(ExtraNetworksPageLora()) if shared.opts.hypernetwork_enabled: diff --git a/modules/ui_javascript.py b/modules/ui_javascript.py index b3aa03cd4..ccf5f8c0d 100644 --- a/modules/ui_javascript.py +++ b/modules/ui_javascript.py @@ -57,8 +57,6 @@ def stylesheet(fn): usercss = os.path.join(data_path, "user.css") if os.path.exists(os.path.join(data_path, "user.css")) else None if modules.shared.opts.theme_type == 'Standard': - if shared.opts.extra_networks_height == 0: - shared.opts.extra_networks_height = 55 themecss = os.path.join(script_path, "javascript", f"{modules.shared.opts.gradio_theme}.css") if os.path.exists(themecss): head += stylesheet(themecss) @@ -66,8 +64,6 @@ def stylesheet(fn): else: modules.shared.log.error(f'UI theme: css="{themecss}" not found') elif modules.shared.opts.theme_type == 'Modern': - if shared.opts.extra_networks_height == 0: - shared.opts.extra_networks_height = 87 theme_folder = next((e.path for e in modules.extensions.extensions if e.name == 'sdnext-modernui'), None) themecss = os.path.join(theme_folder or '', 'themes', f'{modules.shared.opts.gradio_theme}.css') if os.path.exists(themecss): From 70b52664565119e8ddc0df1dbc3f2b42298b865b Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 28 Dec 2024 08:43:35 -0500 Subject: [PATCH 166/249] fix interrogate caption with t5 Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 1 + modules/interrogate.py | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 68d038d58..7a8e946ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ - Fix: live preview image sizes in modern and standard UI - Fix: image width/height calculation when doing img2img - Fix: Flux pipeline switches: txt/img/inpaint +- Fix: Interrogate caption with T5 ## Update for 2024-12-24 diff --git a/modules/interrogate.py b/modules/interrogate.py index 5ae06fb90..6cdaae0c1 100644 --- a/modules/interrogate.py +++ b/modules/interrogate.py @@ -248,14 +248,16 @@ def update_interrogate_params(caption_max_length, chunk_size, min_flavors, max_f def get_clip_models(): import open_clip - return ['/'.join(x) for x in open_clip.list_pretrained()] + models = sorted(open_clip.list_pretrained()) + shared.log.info(f'Interrogate: pkg=openclip version={open_clip.__version__} models={len(models)}') + return ['/'.join(x) for x in models] def load_interrogator(clip_model, blip_model): from installer import install install('clip_interrogator==0.6.0') import clip_interrogator - clip_interrogator.CAPTION_MODELS = caption_models + clip_interrogator.clip_interrogator.CAPTION_MODELS = caption_models global ci # pylint: disable=global-statement if ci is None: interrogator_config = clip_interrogator.Config( @@ -329,6 +331,7 @@ def interrogate_image(image, clip_model, blip_model, mode): except Exception as e: prompt = f"Exception {type(e)}" shared.log.error(f'Interrogate: {e}') + errors.display(e, 'Interrogate') shared.state.end() return prompt From 3d6c7cb92be1d0b2b81f13bbe852f125249f1119 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 28 Dec 2024 16:57:06 -0500 Subject: [PATCH 167/249] quantization improvements and lora support Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 9 ++-- extensions-builtin/Lora/networks.py | 1 - modules/lora/networks.py | 82 ++++++++++++++--------------- modules/model_flux.py | 27 +++++----- modules/model_sana.py | 31 ++++++----- modules/model_sd3.py | 30 ++++++----- modules/shared_state.py | 12 ++--- scripts/hunyuanvideo.py | 16 +++--- scripts/ltxvideo.py | 29 +++++----- 9 files changed, 126 insertions(+), 111 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a8e946ec..aaf0b97ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Change Log for SD.Next -## Update for 2024-12-27 +## Update for 2024-12-28 ### Post release @@ -8,14 +8,17 @@ - **LTXVideo** optimizations: full offload, quantization and tiling support - VAE tiling granular options in *settings -> variable auto encoder* - LoRA loader add legacy option in *settings -> networks* +- LoRA loader better apply for quantized models - Live preview: add sigma calculation, thanks @Disty0 - UI: CSS optimizations when log view is disabled - Sampler options: add flow shift and separate dynamic thresholding from dynamic shifting - Fix: do not show disabled networks - Fix: live preview image sizes in modern and standard UI - Fix: image width/height calculation when doing img2img -- Fix: Flux pipeline switches: txt/img/inpaint -- Fix: Interrogate caption with T5 +- Fix: flux pipeline switches: txt/img/inpaint +- Fix: interrogate caption with T5 +- Fix: on-the-fly quantization using TorchAO +- Fix: remove concurrent preview requests ## Update for 2024-12-24 diff --git a/extensions-builtin/Lora/networks.py b/extensions-builtin/Lora/networks.py index 9f77aac1a..1f02f3846 100644 --- a/extensions-builtin/Lora/networks.py +++ b/extensions-builtin/Lora/networks.py @@ -543,7 +543,6 @@ def network_MultiheadAttention_load_state_dict(self, *args, **kwargs): def list_available_networks(): - print('HERE') t0 = time.time() available_networks.clear() available_network_aliases.clear() diff --git a/modules/lora/networks.py b/modules/lora/networks.py index 2552922fa..28645dd82 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -409,10 +409,40 @@ def network_calc_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn. return batch_updown, batch_ex_bias +def add_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv], weights: Union[None, torch.Tensor] = None, bias: torch.Tensor = None, deactivate: bool = False): + if bias is None: + return self.weight + if deactivate: + bias *= -1 + if weights is None: # weights are used if provided else use self.weight + weights = self.weight + # TODO lora: add other quantization types + if self.__class__.__name__ == 'Linear4bit' and bnb is not None: + try: + dequant_weight = bnb.functional.dequantize_4bit(weights.to(devices.device), quant_state=self.quant_state, quant_type=self.quant_type, blocksize=self.blocksize) + new_weight = dequant_weight.to(devices.device) + bias.to(devices.device) + self.weight = bnb.nn.Params4bit(new_weight, quant_state=self.quant_state, quant_type=self.quant_type, blocksize=self.blocksize) + except Exception as e: + new_weight = None + shared.log.error(f'Load network: type=LoRA quant=bnb type={self.quant_type} blocksize={self.blocksize} state={vars(self.quant_state)} weight={self.weight} bias={bias} {e}') + else: + try: + new_weight = weights.to(devices.device) + bias.to(devices.device) + except Exception: + new_weight = weights + bias # try without device cast + self.weight = torch.nn.Parameter(new_weight, requires_grad=False) + del weights + try: + self.weight = self.weight.to(device=devices.device) # required since quantization happens only during .to call, not during params creation + except Exception: + pass # may fail if weights is meta tensor + return self.weight + + def network_apply_direct(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv], updown: torch.Tensor, ex_bias: torch.Tensor, deactivate: bool = False): weights_backup = getattr(self, "network_weights_backup", False) bias_backup = getattr(self, "network_bias_backup", False) - if not isinstance(weights_backup, bool): + if not isinstance(weights_backup, bool): # remove previous backup if we switched settings weights_backup = True if not isinstance(bias_backup, bool): bias_backup = True @@ -424,34 +454,14 @@ def network_apply_direct(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn. if updown is not None and len(self.weight.shape) == 4 and self.weight.shape[1] == 9: # inpainting model. zero pad updown to make channel[1] 4 to 9 updown = torch.nn.functional.pad(updown, (0, 0, 0, 0, 0, 5)) # pylint: disable=not-callable if updown is not None: - if deactivate: - updown *= -1 - if getattr(self, "quant_type", None) in ['nf4', 'fp4'] and bnb is not None: - try: # TODO lora load: direct with bnb - weight = bnb.functional.dequantize_4bit(self.weight, quant_state=self.quant_state, quant_type=self.quant_type, blocksize=self.blocksize) - new_weight = weight.to(devices.device) + updown.to(devices.device) - self.weight = bnb.nn.Params4bit(new_weight, quant_state=self.quant_state, quant_type=self.quant_type, blocksize=self.blocksize) - except Exception: - # shared.log.error(f'Load network: type=LoRA quant=bnb type={self.quant_type} state={self.quant_state} blocksize={self.blocksize} {e}') - extra_network_lora.errors['bnb'] = extra_network_lora.errors.get('bnb', 0) + 1 - new_weight = None - else: - try: - new_weight = self.weight.to(devices.device) + updown.to(devices.device) - except Exception: - new_weight = self.weight + updown - self.weight = torch.nn.Parameter(new_weight, requires_grad=False) - del new_weight - if hasattr(self, "qweight") and hasattr(self, "freeze"): - self.freeze() + self.weight = add_weights(self, bias=updown, deactivate=deactivate) if bias_backup: if ex_bias is not None: - if deactivate: - ex_bias *= -1 - new_weight = bias_backup.to(devices.device) + ex_bias.to(devices.device) - self.bias = torch.nn.Parameter(new_weight, requires_grad=False) - del new_weight + self.bias = add_weights(self, bias=ex_bias, deactivate=deactivate) + + if hasattr(self, "qweight") and hasattr(self, "freeze"): + self.freeze() timer.apply += time.time() - t0 return self.weight.device, self.weight.dtype @@ -469,30 +479,20 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn if updown is not None and len(weights_backup.shape) == 4 and weights_backup.shape[1] == 9: # inpainting model. zero pad updown to make channel[1] 4 to 9 updown = torch.nn.functional.pad(updown, (0, 0, 0, 0, 0, 5)) # pylint: disable=not-callable if updown is not None: - if deactivate: - updown *= -1 - new_weight = weights_backup.to(devices.device) + updown.to(devices.device) - if getattr(self, "quant_type", None) in ['nf4', 'fp4'] and bnb is not None: - self.weight = bnb.nn.Params4bit(new_weight, quant_state=self.quant_state, quant_type=self.quant_type, blocksize=self.blocksize) - else: - self.weight = torch.nn.Parameter(new_weight.to(device=orig_device), requires_grad=False) - del new_weight + self.weight = add_weights(self, weights=weights_backup, bias=updown, deactivate=deactivate) else: self.weight = torch.nn.Parameter(weights_backup.to(device=orig_device), requires_grad=False) - if hasattr(self, "qweight") and hasattr(self, "freeze"): - self.freeze() if bias_backup is not None: self.bias = None if ex_bias is not None: - if deactivate: - ex_bias *= -1 - new_weight = bias_backup.to(devices.device) + ex_bias.to(devices.device) - self.bias = torch.nn.Parameter(new_weight.to(device=orig_device), requires_grad=False) - del new_weight + self.weight = add_weights(self, weights=weights_backup, bias=ex_bias, deactivate=deactivate) else: self.bias = torch.nn.Parameter(bias_backup.to(device=orig_device), requires_grad=False) + if hasattr(self, "qweight") and hasattr(self, "freeze"): + self.freeze() + timer.apply += time.time() - t0 return self.weight.device, self.weight.dtype diff --git a/modules/model_flux.py b/modules/model_flux.py index f2286866e..dd0a507db 100644 --- a/modules/model_flux.py +++ b/modules/model_flux.py @@ -144,19 +144,21 @@ def quant_flux_bnb(checkpoint_info, transformer, text_encoder_2): def load_quants(kwargs, repo_id, cache_dir): - if len(shared.opts.bnb_quantization) > 0: - quant_args = {} - quant_args = model_quant.create_bnb_config(quant_args) - quant_args = model_quant.create_ao_config(quant_args) - if not quant_args: - return kwargs + quant_args = {} + quant_args = model_quant.create_bnb_config(quant_args) + if quant_args: model_quant.load_bnb(f'Load model: type=FLUX quant={quant_args}') - if 'Model' in shared.opts.bnb_quantization and 'transformer' not in kwargs: - kwargs['transformer'] = diffusers.FluxTransformer2DModel.from_pretrained(repo_id, subfolder="transformer", cache_dir=cache_dir, torch_dtype=devices.dtype, **quant_args) - shared.log.debug(f'Quantization: module=transformer type=bnb dtype={shared.opts.bnb_quantization_type} storage={shared.opts.bnb_quantization_storage}') - if 'Text Encoder' in shared.opts.bnb_quantization and 'text_encoder_3' not in kwargs: - kwargs['text_encoder_2'] = transformers.T5EncoderModel.from_pretrained(repo_id, subfolder="text_encoder_2", cache_dir=cache_dir, torch_dtype=devices.dtype, **quant_args) - shared.log.debug(f'Quantization: module=t5 type=bnb dtype={shared.opts.bnb_quantization_type} storage={shared.opts.bnb_quantization_storage}') + quant_args = model_quant.create_ao_config(quant_args) + if quant_args: + model_quant.load_torchao(f'Load model: type=FLUX quant={quant_args}') + if not quant_args: + return kwargs + if 'transformer' not in kwargs and ('Model' in shared.opts.bnb_quantization or 'Model' in shared.opts.torchao_quantization): + kwargs['transformer'] = diffusers.FluxTransformer2DModel.from_pretrained(repo_id, subfolder="transformer", cache_dir=cache_dir, torch_dtype=devices.dtype, **quant_args) + shared.log.debug(f'Quantization: module=transformer type=bnb dtype={shared.opts.bnb_quantization_type} storage={shared.opts.bnb_quantization_storage}') + if 'text_encoder_3' not in kwargs and ('Text Encoder' in shared.opts.bnb_quantization or 'Text Encoder' in shared.opts.torchao_quantization): + kwargs['text_encoder_2'] = transformers.T5EncoderModel.from_pretrained(repo_id, subfolder="text_encoder_2", cache_dir=cache_dir, torch_dtype=devices.dtype, **quant_args) + shared.log.debug(f'Quantization: module=t5 type=bnb dtype={shared.opts.bnb_quantization_type} storage={shared.opts.bnb_quantization_storage}') return kwargs @@ -353,7 +355,6 @@ def load_flux(checkpoint_info, diffusers_load_config): # triggered by opts.sd_ch allow_quant = 'gguf' not in (sd_unet.loaded_unet or '') fn = checkpoint_info.path if (fn is None) or (not os.path.exists(fn) or os.path.isdir(fn)): - # transformer, text_encoder_2 = quant_flux_bnb(checkpoint_info, transformer, text_encoder_2) kwargs = load_quants(kwargs, repo_id, cache_dir=shared.opts.diffusers_dir) kwargs = model_quant.create_bnb_config(kwargs, allow_quant) kwargs = model_quant.create_ao_config(kwargs, allow_quant) diff --git a/modules/model_sana.py b/modules/model_sana.py index b9f56c7c6..792b49d15 100644 --- a/modules/model_sana.py +++ b/modules/model_sana.py @@ -7,20 +7,23 @@ def load_quants(kwargs, repo_id, cache_dir): - if len(shared.opts.bnb_quantization) > 0: - quant_args = {} - quant_args = model_quant.create_bnb_config(quant_args) - quant_args = model_quant.create_ao_config(quant_args) - load_args = kwargs.copy() - if not quant_args: - return kwargs - model_quant.load_bnb(f'Load model: type=SD3 quant={quant_args} args={load_args}') - if 'Model' in shared.opts.bnb_quantization and 'transformer' not in kwargs: - kwargs['transformer'] = diffusers.models.SanaTransformer2DModel.from_pretrained(repo_id, subfolder="transformer", cache_dir=cache_dir, **load_args, **quant_args) - shared.log.debug(f'Quantization: module=transformer type=bnb dtype={shared.opts.bnb_quantization_type} storage={shared.opts.bnb_quantization_storage}') - if 'Text Encoder' in shared.opts.bnb_quantization and 'text_encoder_3' not in kwargs: - kwargs['text_encoder_3'] = transformers.AutoModelForCausalLM.from_pretrained(repo_id, subfolder="text_encoder", cache_dir=cache_dir, **load_args, **quant_args) - shared.log.debug(f'Quantization: module=t5 type=bnb dtype={shared.opts.bnb_quantization_type} storage={shared.opts.bnb_quantization_storage}') + quant_args = {} + quant_args = model_quant.create_bnb_config(quant_args) + if quant_args: + model_quant.load_bnb(f'Load model: type=Sana quant={quant_args}') + quant_args = model_quant.create_ao_config(quant_args) + if quant_args: + model_quant.load_torchao(f'Load model: type=Sana quant={quant_args}') + if not quant_args: + return kwargs + load_args = kwargs.copy() + model_quant.load_bnb(f'Load model: type=SD3 quant={quant_args} args={load_args}') + if 'Model' in shared.opts.bnb_quantization and 'transformer' not in kwargs: + kwargs['transformer'] = diffusers.models.SanaTransformer2DModel.from_pretrained(repo_id, subfolder="transformer", cache_dir=cache_dir, **load_args, **quant_args) + shared.log.debug(f'Quantization: module=transformer type=bnb dtype={shared.opts.bnb_quantization_type} storage={shared.opts.bnb_quantization_storage}') + if 'Text Encoder' in shared.opts.bnb_quantization and 'text_encoder_3' not in kwargs: + kwargs['text_encoder_3'] = transformers.AutoModelForCausalLM.from_pretrained(repo_id, subfolder="text_encoder", cache_dir=cache_dir, **load_args, **quant_args) + shared.log.debug(f'Quantization: module=t5 type=bnb dtype={shared.opts.bnb_quantization_type} storage={shared.opts.bnb_quantization_storage}') return kwargs diff --git a/modules/model_sd3.py b/modules/model_sd3.py index 1834c573e..df9a5f780 100644 --- a/modules/model_sd3.py +++ b/modules/model_sd3.py @@ -42,29 +42,31 @@ def load_overrides(kwargs, cache_dir): from modules import sd_vae vae_file = sd_vae.vae_dict[shared.opts.sd_vae] if os.path.exists(vae_file): - vae_config = os.path.join('configs', 'flux', 'vae', 'config.json') + vae_config = os.path.join('configs', 'sd3', 'vae', 'config.json') kwargs['vae'] = diffusers.AutoencoderKL.from_single_file(vae_file, config=vae_config, cache_dir=cache_dir, torch_dtype=devices.dtype) shared.log.debug(f'Load model: type=SD3 vae="{shared.opts.sd_vae}"') except Exception as e: - shared.log.error(f"Load model: type=FLUX failed to load VAE: {e}") + shared.log.error(f"Load model: type=SD3 failed to load VAE: {e}") shared.opts.sd_vae = 'None' return kwargs def load_quants(kwargs, repo_id, cache_dir): - if len(shared.opts.bnb_quantization) > 0: - quant_args = {} - quant_args = model_quant.create_bnb_config(quant_args) - quant_args = model_quant.create_ao_config(quant_args) - if not quant_args: - return kwargs + quant_args = {} + quant_args = model_quant.create_bnb_config(quant_args) + if quant_args: model_quant.load_bnb(f'Load model: type=SD3 quant={quant_args}') - if 'Model' in shared.opts.bnb_quantization and 'transformer' not in kwargs: - kwargs['transformer'] = diffusers.SD3Transformer2DModel.from_pretrained(repo_id, subfolder="transformer", cache_dir=cache_dir, torch_dtype=devices.dtype, **quant_args) - shared.log.debug(f'Quantization: module=transformer type=bnb dtype={shared.opts.bnb_quantization_type} storage={shared.opts.bnb_quantization_storage}') - if 'Text Encoder' in shared.opts.bnb_quantization and 'text_encoder_3' not in kwargs: - kwargs['text_encoder_3'] = transformers.T5EncoderModel.from_pretrained(repo_id, subfolder="text_encoder_3", variant='fp16', cache_dir=cache_dir, torch_dtype=devices.dtype, **quant_args) - shared.log.debug(f'Quantization: module=t5 type=bnb dtype={shared.opts.bnb_quantization_type} storage={shared.opts.bnb_quantization_storage}') + quant_args = model_quant.create_ao_config(quant_args) + if quant_args: + model_quant.load_torchao(f'Load model: type=SD3 quant={quant_args}') + if not quant_args: + return kwargs + if 'Model' in shared.opts.bnb_quantization and 'transformer' not in kwargs: + kwargs['transformer'] = diffusers.SD3Transformer2DModel.from_pretrained(repo_id, subfolder="transformer", cache_dir=cache_dir, torch_dtype=devices.dtype, **quant_args) + shared.log.debug(f'Quantization: module=transformer type=bnb dtype={shared.opts.bnb_quantization_type} storage={shared.opts.bnb_quantization_storage}') + if 'Text Encoder' in shared.opts.bnb_quantization and 'text_encoder_3' not in kwargs: + kwargs['text_encoder_3'] = transformers.T5EncoderModel.from_pretrained(repo_id, subfolder="text_encoder_3", variant='fp16', cache_dir=cache_dir, torch_dtype=devices.dtype, **quant_args) + shared.log.debug(f'Quantization: module=t5 type=bnb dtype={shared.opts.bnb_quantization_type} storage={shared.opts.bnb_quantization_storage}') return kwargs diff --git a/modules/shared_state.py b/modules/shared_state.py index b375a42e9..c773b36a9 100644 --- a/modules/shared_state.py +++ b/modules/shared_state.py @@ -27,6 +27,7 @@ class State: prediction_type = "epsilon" api = False disable_preview = False + preview_busy = False time_start = None need_restart = False server_start = time.time() @@ -156,26 +157,25 @@ def set_current_image(self): self.do_set_current_image() def do_set_current_image(self): - if self.current_latent is None: + if self.current_latent is None or self.disable_preview or self.preview_busy: return - from modules.shared import opts - import modules.sd_samplers # pylint: disable=W0621 + from modules import shared, sd_samplers + self.preview_busy = True try: sample = self.current_latent self.current_image_sampling_step = self.sampling_step - if self.disable_preview: - return if self.current_noise_pred is not None and self.current_sigma is not None and self.current_sigma_next is not None: original_sample = sample - (self.current_noise_pred * (self.current_sigma_next-self.current_sigma)) if self.prediction_type in {"epsilon", "flow_prediction"}: sample = original_sample - (self.current_noise_pred * self.current_sigma) elif self.prediction_type == "v_prediction": sample = self.current_noise_pred * (-self.current_sigma / (self.current_sigma**2 + 1) ** 0.5) + (original_sample / (self.current_sigma**2 + 1)) # pylint: disable=invalid-unary-operand-type - image = modules.sd_samplers.samples_to_image_grid(sample) if opts.show_progress_grid else modules.sd_samplers.sample_to_image(sample) + image = sd_samplers.samples_to_image_grid(sample) if shared.opts.show_progress_grid else sd_samplers.sample_to_image(sample) self.assign_current_image(image) except Exception: # log.error(f'Error setting current image: step={self.sampling_step} {e}') pass + self.preview_busy = False def assign_current_image(self, image): self.current_image = image diff --git a/scripts/hunyuanvideo.py b/scripts/hunyuanvideo.py index 61ef07f30..d4ceee53e 100644 --- a/scripts/hunyuanvideo.py +++ b/scripts/hunyuanvideo.py @@ -100,16 +100,20 @@ def run(self, p: processing.StableDiffusionProcessing, num_frames, tile_frames, if shared.sd_model.__class__ != diffusers.HunyuanVideoPipeline: sd_models.unload_model_weights() t0 = time.time() - kwargs = {} - kwargs = model_quant.create_bnb_config(kwargs) - kwargs = model_quant.create_ao_config(kwargs) + quant_args = {} + quant_args = model_quant.create_bnb_config(quant_args) + if quant_args: + model_quant.load_bnb(f'Load model: type=HunyuanVideo quant={quant_args}') + quant_args = model_quant.create_ao_config(quant_args) + if quant_args: + model_quant.load_torchao(f'Load model: type=HunyuanVideo quant={quant_args}') transformer = diffusers.HunyuanVideoTransformer3DModel.from_pretrained( repo_id, subfolder="transformer", torch_dtype=devices.dtype, revision="refs/pr/18", cache_dir = shared.opts.hfcache_dir, - **kwargs + **quant_args ) shared.log.debug(f'Video: module={transformer.__class__.__name__}') text_encoder = transformers.LlamaModel.from_pretrained( @@ -118,7 +122,7 @@ def run(self, p: processing.StableDiffusionProcessing, num_frames, tile_frames, revision="refs/pr/18", cache_dir = shared.opts.hfcache_dir, torch_dtype=devices.dtype, - **kwargs + **quant_args ) shared.log.debug(f'Video: module={text_encoder.__class__.__name__}') shared.sd_model = diffusers.HunyuanVideoPipeline.from_pretrained( @@ -128,7 +132,7 @@ def run(self, p: processing.StableDiffusionProcessing, num_frames, tile_frames, revision="refs/pr/18", cache_dir = shared.opts.hfcache_dir, torch_dtype=devices.dtype, - **kwargs + **quant_args ) t1 = time.time() shared.log.debug(f'Video: load cls={shared.sd_model.__class__.__name__} repo="{repo_id}" dtype={devices.dtype} time={t1-t0:.2f}') diff --git a/scripts/ltxvideo.py b/scripts/ltxvideo.py index ab2e3ed95..279d98d92 100644 --- a/scripts/ltxvideo.py +++ b/scripts/ltxvideo.py @@ -15,19 +15,22 @@ def load_quants(kwargs, repo_id): - if len(shared.opts.bnb_quantization) > 0: - quant_args = {} - quant_args = model_quant.create_bnb_config(quant_args) - quant_args = model_quant.create_ao_config(quant_args) - if not quant_args: - return kwargs - model_quant.load_bnb(f'Load model: type=LTX quant={quant_args}') - if 'Model' in shared.opts.bnb_quantization and 'transformer' not in kwargs: - kwargs['transformer'] = diffusers.LTXVideoTransformer3DModel.from_pretrained(repo_id, subfolder="transformer", cache_dir=shared.opts.hfcache_dir, torch_dtype=devices.dtype, **quant_args) - shared.log.debug(f'Quantization: module=transformer type=bnb dtype={shared.opts.bnb_quantization_type} storage={shared.opts.bnb_quantization_storage}') - if 'Text Encoder' in shared.opts.bnb_quantization and 'text_encoder_3' not in kwargs: - kwargs['text_encoder'] = transformers.T5EncoderModel.from_pretrained(repo_id, subfolder="text_encoder", cache_dir=shared.opts.hfcache_dir, torch_dtype=devices.dtype, **quant_args) - shared.log.debug(f'Quantization: module=t5 type=bnb dtype={shared.opts.bnb_quantization_type} storage={shared.opts.bnb_quantization_storage}') + quant_args = {} + quant_args = model_quant.create_bnb_config(quant_args) + if quant_args: + model_quant.load_bnb(f'Load model: type=LTXVideo quant={quant_args}') + quant_args = model_quant.create_ao_config(quant_args) + if quant_args: + model_quant.load_torchao(f'Load model: type=LTXVideo quant={quant_args}') + if not quant_args: + return kwargs + model_quant.load_bnb(f'Load model: type=LTX quant={quant_args}') + if 'Model' in shared.opts.bnb_quantization and 'transformer' not in kwargs: + kwargs['transformer'] = diffusers.LTXVideoTransformer3DModel.from_pretrained(repo_id, subfolder="transformer", cache_dir=shared.opts.hfcache_dir, torch_dtype=devices.dtype, **quant_args) + shared.log.debug(f'Quantization: module=transformer type=bnb dtype={shared.opts.bnb_quantization_type} storage={shared.opts.bnb_quantization_storage}') + if 'Text Encoder' in shared.opts.bnb_quantization and 'text_encoder_3' not in kwargs: + kwargs['text_encoder'] = transformers.T5EncoderModel.from_pretrained(repo_id, subfolder="text_encoder", cache_dir=shared.opts.hfcache_dir, torch_dtype=devices.dtype, **quant_args) + shared.log.debug(f'Quantization: module=t5 type=bnb dtype={shared.opts.bnb_quantization_type} storage={shared.opts.bnb_quantization_storage}') return kwargs From ea6282ccb20a5b81a3035e20039620cc09331330 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 28 Dec 2024 17:04:43 -0500 Subject: [PATCH 168/249] cleanup Signed-off-by: Vladimir Mandic --- modules/lora/networks.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/modules/lora/networks.py b/modules/lora/networks.py index 28645dd82..885205570 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -414,7 +414,7 @@ def add_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm return self.weight if deactivate: bias *= -1 - if weights is None: # weights are used if provided else use self.weight + if weights is None: # weights are used if provided-from-backup else use self.weight weights = self.weight # TODO lora: add other quantization types if self.__class__.__name__ == 'Linear4bit' and bnb is not None: @@ -423,15 +423,13 @@ def add_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm new_weight = dequant_weight.to(devices.device) + bias.to(devices.device) self.weight = bnb.nn.Params4bit(new_weight, quant_state=self.quant_state, quant_type=self.quant_type, blocksize=self.blocksize) except Exception as e: - new_weight = None - shared.log.error(f'Load network: type=LoRA quant=bnb type={self.quant_type} blocksize={self.blocksize} state={vars(self.quant_state)} weight={self.weight} bias={bias} {e}') + shared.log.error(f'Load network: type=LoRA quant=bnb cls={self.__class__.__name__} type={self.quant_type} blocksize={self.blocksize} state={vars(self.quant_state)} weight={self.weight} bias={bias} {e}') else: try: new_weight = weights.to(devices.device) + bias.to(devices.device) except Exception: new_weight = weights + bias # try without device cast self.weight = torch.nn.Parameter(new_weight, requires_grad=False) - del weights try: self.weight = self.weight.to(device=devices.device) # required since quantization happens only during .to call, not during params creation except Exception: From 247845648e9a0e79c57a5bf0b3700fadf636dd72 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 29 Dec 2024 12:38:19 -0500 Subject: [PATCH 169/249] lora direct on-demand apply/unapply Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 30 ++++++++------- modules/devices.py | 1 + modules/extra_networks.py | 17 ++++++--- modules/lora/extra_networks_lora.py | 57 ++++++++++++++++++++++++----- modules/lora/networks.py | 44 +++++++++++++--------- modules/model_flux.py | 7 ++-- modules/model_sana.py | 7 ++-- modules/model_sd3.py | 7 ++-- modules/processing_args.py | 10 +++-- modules/processing_diffusers.py | 5 +-- modules/processing_helpers.py | 3 +- modules/processing_vae.py | 6 +-- modules/sd_models.py | 4 +- modules/sd_samplers.py | 6 +-- modules/styles.py | 6 +-- scripts/hunyuanvideo.py | 7 ++-- scripts/ltxvideo.py | 7 ++-- 17 files changed, 142 insertions(+), 82 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aaf0b97ed..fe2edffe8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,24 +1,26 @@ # Change Log for SD.Next -## Update for 2024-12-28 - -### Post release +## Update for 2024-12-29 +- **LoRA**: + - **Sana** support + - quantized models support + - fuse support with on-demand apply/unapply + - add legacy option in *settings -> networks* - **HunyuanVideo** optimizations: full offload, quantization and tiling support - **LTXVideo** optimizations: full offload, quantization and tiling support - VAE tiling granular options in *settings -> variable auto encoder* -- LoRA loader add legacy option in *settings -> networks* -- LoRA loader better apply for quantized models -- Live preview: add sigma calculation, thanks @Disty0 +- UI: live preview add sigma calculation, thanks @Disty0 - UI: CSS optimizations when log view is disabled -- Sampler options: add flow shift and separate dynamic thresholding from dynamic shifting -- Fix: do not show disabled networks -- Fix: live preview image sizes in modern and standard UI -- Fix: image width/height calculation when doing img2img -- Fix: flux pipeline switches: txt/img/inpaint -- Fix: interrogate caption with T5 -- Fix: on-the-fly quantization using TorchAO -- Fix: remove concurrent preview requests +- Samplers: add flow shift options and separate dynamic thresholding from dynamic shifting +- **Fixes** + - do not show disabled networks + - live preview image sizes in modern and standard UI + - image width/height calculation when doing img2img + - flux pipeline switches: txt/img/inpaint + - interrogate caption with T5 + - on-the-fly quantization using TorchAO + - remove concurrent preview requests ## Update for 2024-12-24 diff --git a/modules/devices.py b/modules/devices.py index 949fab4aa..6168ac63a 100644 --- a/modules/devices.py +++ b/modules/devices.py @@ -516,6 +516,7 @@ def randn_without_seed(shape): return torch.randn(shape, device=cpu).to(device) return torch.randn(shape, device=device) + def autocast(disable=False): if disable or dtype == torch.float32: return contextlib.nullcontext() diff --git a/modules/extra_networks.py b/modules/extra_networks.py index 8bd742ce0..25b366e05 100644 --- a/modules/extra_networks.py +++ b/modules/extra_networks.py @@ -80,14 +80,14 @@ def activate(p, extra_network_data=None, step=0, include=[], exclude=[]): if p.disable_extra_networks: return extra_network_data = extra_network_data or p.network_data - if extra_network_data is None or len(extra_network_data) == 0: - return + # if extra_network_data is None or len(extra_network_data) == 0: + # return stepwise = False for extra_network_args in extra_network_data.values(): stepwise = stepwise or is_stepwise(extra_network_args) functional = shared.opts.lora_functional if shared.opts.lora_force_diffusers and stepwise: - shared.log.warning("Composable LoRA not compatible with 'lora_force_diffusers'") + shared.log.warning("Load network: type=LoRA method=composable loader=diffusers not compatible") stepwise = False shared.opts.data['lora_functional'] = stepwise or functional @@ -110,7 +110,12 @@ def activate(p, extra_network_data=None, step=0, include=[], exclude=[]): if args is not None: continue try: - extra_network.activate(p, []) + # extra_network.activate(p, []) + signature = list(inspect.signature(extra_network.activate).parameters) + if 'include' in signature and 'exclude' in signature: + extra_network.activate(p, [], include=include, exclude=exclude) + else: + extra_network.activate(p, []) except Exception as e: errors.display(e, f"Activating network: type={extra_network_name}") @@ -125,8 +130,8 @@ def deactivate(p, extra_network_data=None): if p.disable_extra_networks: return extra_network_data = extra_network_data or p.network_data - if extra_network_data is None or len(extra_network_data) == 0: - return + # if extra_network_data is None or len(extra_network_data) == 0: + # return for extra_network_name in extra_network_data: extra_network = extra_network_registry.get(extra_network_name, None) if extra_network is None: diff --git a/modules/lora/extra_networks_lora.py b/modules/lora/extra_networks_lora.py index 42c4a92f6..9708fb170 100644 --- a/modules/lora/extra_networks_lora.py +++ b/modules/lora/extra_networks_lora.py @@ -1,11 +1,16 @@ +from typing import List +import os import re import numpy as np -import modules.lora.networks as networks +from modules.lora import networks from modules import extra_networks, shared -# from https://github.com/cheald/sd-webui-loractl/blob/master/loractl/lib/utils.py -def get_stepwise(param, step, steps): +debug = os.environ.get('SD_SCRIPT_DEBUG', None) is not None +debug_log = shared.log.trace if debug else lambda *args, **kwargs: None + + +def get_stepwise(param, step, steps): # from https://github.com/cheald/sd-webui-loractl/blob/master/loractl/lib/utils.py def sorted_positions(raw_steps): steps = [[float(s.strip()) for s in re.split("[@~]", x)] for x in re.split("[,;]", str(raw_steps))] @@ -46,7 +51,8 @@ def prompt(p): if len(all_tags) > 0: all_tags = list(set(all_tags)) all_tags = [t for t in all_tags if t not in p.prompt] - shared.log.debug(f"Load network: type=LoRA tags={all_tags} max={shared.opts.lora_apply_tags} apply") + if len(all_tags) > 0: + shared.log.debug(f"Load network: type=LoRA tags={all_tags} max={shared.opts.lora_apply_tags} apply") all_tags = ', '.join(all_tags) p.extra_generation_params["LoRA tags"] = all_tags if '_tags_' in p.prompt: @@ -114,26 +120,58 @@ def __init__(self): self.model = None self.errors = {} + def signature(self, names: List[str], te_multipliers: List, unet_multipliers: List): + return [f'{name}:{te}:{unet}' for name, te, unet in zip(names, te_multipliers, unet_multipliers)] + + def changed(self, requested: List[str], include: List[str], exclude: List[str]): + sd_model = getattr(shared.sd_model, "pipe", shared.sd_model) + if not hasattr(sd_model, 'loaded_loras'): + sd_model.loaded_loras = {} + key = f'{','.join(include)}:{','.join(exclude)}' + loaded = sd_model.loaded_loras.get(key, []) + # shared.log.trace(f'Load network: type=LoRA key="{key}" requested={requested} loaded={loaded}') + if len(requested) != len(loaded): + sd_model.loaded_loras[key] = requested + return True + for r, l in zip(requested, loaded): + if r != l: + sd_model.loaded_loras[key] = requested + return True + return False + def activate(self, p, params_list, step=0, include=[], exclude=[]): self.errors.clear() if self.active: if self.model != shared.opts.sd_model_checkpoint: # reset if model changed self.active = False if len(params_list) > 0 and not self.active: # activate patches once - # shared.log.debug(f'Activate network: type=LoRA model="{shared.opts.sd_model_checkpoint}"') self.active = True self.model = shared.opts.sd_model_checkpoint - if 'text_encoder' in include: - networks.timer.clear(complete=True) names, te_multipliers, unet_multipliers, dyn_dims = parse(p, params_list, step) + requested = self.signature(names, te_multipliers, unet_multipliers) + + if debug: + import sys + fn = f'{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access + debug_log(f'Load network: type=LoRA include={include} exclude={exclude} requested={requested} fn={fn}') + networks.network_load(names, te_multipliers, unet_multipliers, dyn_dims) # load - networks.network_activate(include, exclude) + has_changed = self.changed(requested, include, exclude) + if has_changed: + networks.network_deactivate(include, exclude) + networks.network_activate(include, exclude) + debug_log(f'Load network: type=LoRA previous={[n.name for n in networks.previously_loaded_networks]} current={[n.name for n in networks.loaded_networks]} changed') + if len(networks.loaded_networks) > 0 and len(networks.applied_layers) > 0 and step == 0: infotext(p) prompt(p) - shared.log.info(f'Load network: type=LoRA apply={[n.name for n in networks.loaded_networks]} mode={"fuse" if shared.opts.lora_fuse_diffusers else "backup"} te={te_multipliers} unet={unet_multipliers} time={networks.timer.summary}') + if has_changed and len(include) == 0: # print only once + shared.log.info(f'Load network: type=LoRA apply={[n.name for n in networks.loaded_networks]} mode={"fuse" if shared.opts.lora_fuse_diffusers else "backup"} te={te_multipliers} unet={unet_multipliers} time={networks.timer.summary}') def deactivate(self, p): + if shared.native: + networks.previously_loaded_networks = networks.loaded_networks.copy() + debug_log(f'Load network: type=LoRA active={[n.name for n in networks.previously_loaded_networks]} deactivate') if shared.native and len(networks.diffuser_loaded) > 0: if hasattr(shared.sd_model, "unload_lora_weights") and hasattr(shared.sd_model, "text_encoder"): if not (shared.compiled_model_state is not None and shared.compiled_model_state.is_compiled is True): @@ -143,7 +181,6 @@ def deactivate(self, p): shared.sd_model.unload_lora_weights() # fails for non-CLIP models except Exception: pass - networks.network_deactivate() if self.active and networks.debug: shared.log.debug(f"Network end: type=LoRA time={networks.timer.summary}") if self.errors: diff --git a/modules/lora/networks.py b/modules/lora/networks.py index 885205570..7863e943b 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -19,6 +19,7 @@ available_networks = {} available_network_aliases = {} loaded_networks: List[network.Network] = [] +previously_loaded_networks: List[network.Network] = [] applied_layers: list[str] = [] bnb = None lora_cache = {} @@ -286,7 +287,7 @@ def network_load(names, te_multipliers=None, unet_multipliers=None, dyn_dims=Non errors.display(e, 'LoRA') if len(loaded_networks) > 0 and debug: - shared.log.debug(f'Load network: type=LoRA loaded={len(loaded_networks)} cache={list(lora_cache)}') + shared.log.debug(f'Load network: type=LoRA loaded={[n.name for n in loaded_networks]} cache={list(lora_cache)}') if recompile_model: shared.log.info("Load network: type=LoRA recompiling model") @@ -362,7 +363,7 @@ def network_backup_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.n return backup_size -def network_calc_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv], network_layer_name: str): +def network_calc_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv], network_layer_name: str, use_previous: bool = False): if shared.opts.diffusers_offload_mode == "none": try: self.to(devices.device) @@ -370,7 +371,8 @@ def network_calc_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn. pass batch_updown = None batch_ex_bias = None - for net in loaded_networks: + loaded = loaded_networks if not use_previous else previously_loaded_networks + for net in loaded: module = net.modules.get(network_layer_name, None) if module is None: continue @@ -495,22 +497,26 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn return self.weight.device, self.weight.dtype -def network_deactivate(): +def network_deactivate(include=[], exclude=[]): if not shared.opts.lora_fuse_diffusers: return t0 = time.time() - timer.clear() sd_model = getattr(shared.sd_model, "pipe", shared.sd_model) # wrapped model compatiblility if shared.opts.diffusers_offload_mode == "sequential": sd_models.disable_offload(sd_model) sd_models.move_model(sd_model, device=devices.cpu) modules = {} - for component_name in ['text_encoder', 'text_encoder_2', 'unet', 'transformer']: - component = getattr(sd_model, component_name, None) + + components = include if len(include) > 0 else ['text_encoder', 'text_encoder_2', 'text_encoder_3', 'unet', 'transformer'] + components = [x for x in components if x not in exclude] + active_components = [] + for name in components: + component = getattr(sd_model, name, None) if component is not None and hasattr(component, 'named_modules'): - modules[component_name] = list(component.named_modules()) + modules[name] = list(component.named_modules()) + active_components.append(name) total = sum(len(x) for x in modules.values()) - if len(loaded_networks) > 0: + if len(previously_loaded_networks) > 0 and debug: pbar = rp.Progress(rp.TextColumn('[cyan]Network: type=LoRA action=deactivate'), rp.BarColumn(), rp.TaskProgressColumn(), rp.TimeRemainingColumn(), rp.TimeElapsedColumn(), rp.TextColumn('[cyan]{task.description}'), console=shared.console) task = pbar.add_task(description='', total=total) else: @@ -528,7 +534,7 @@ def network_deactivate(): if task is not None: pbar.update(task, advance=1) continue - batch_updown, batch_ex_bias = network_calc_weights(module, network_layer_name) + batch_updown, batch_ex_bias = network_calc_weights(module, network_layer_name, use_previous=True) if shared.opts.lora_fuse_diffusers: weights_device, weights_dtype = network_apply_direct(module, batch_updown, batch_ex_bias, deactivate=True) else: @@ -540,11 +546,12 @@ def network_deactivate(): del batch_updown, batch_ex_bias module.network_current_names = () if task is not None: - pbar.update(task, advance=1, description=f'networks={len(loaded_networks)} modules={len(modules)} deactivate={len(applied_layers)}') - weights_devices, weights_dtypes = list(set([x for x in weights_devices if x is not None])), list(set([x for x in weights_dtypes if x is not None])) # noqa: C403 # pylint: disable=R1718 + pbar.update(task, advance=1, description=f'networks={len(previously_loaded_networks)} modules={active_components} layers={total} unapply={len(applied_layers)}') + timer.deactivate = time.time() - t0 - if debug and len(loaded_networks) > 0: - shared.log.debug(f'Deactivate network: type=LoRA networks={len(loaded_networks)} modules={total} deactivate={len(applied_layers)} device={weights_devices} dtype={weights_dtypes} fuse={shared.opts.lora_fuse_diffusers} time={timer.summary}') + if debug and len(previously_loaded_networks) > 0: + weights_devices, weights_dtypes = list(set([x for x in weights_devices if x is not None])), list(set([x for x in weights_dtypes if x is not None])) # noqa: C403 # pylint: disable=R1718 + shared.log.debug(f'Deactivate network: type=LoRA networks={[n.name for n in previously_loaded_networks]} modules={active_components} layers={total} apply={len(applied_layers)} device={weights_devices} dtype={weights_dtypes} fuse={shared.opts.lora_fuse_diffusers} time={timer.summary}') modules.clear() if shared.opts.diffusers_offload_mode == "sequential": sd_models.set_diffuser_offload(sd_model, op="model") @@ -559,9 +566,11 @@ def network_activate(include=[], exclude=[]): modules = {} components = include if len(include) > 0 else ['text_encoder', 'text_encoder_2', 'text_encoder_3', 'unet', 'transformer'] components = [x for x in components if x not in exclude] + active_components = [] for name in components: component = getattr(sd_model, name, None) if component is not None and hasattr(component, 'named_modules'): + active_components.append(name) modules[name] = list(component.named_modules()) total = sum(len(x) for x in modules.values()) if len(loaded_networks) > 0: @@ -598,13 +607,14 @@ def network_activate(include=[], exclude=[]): del batch_updown, batch_ex_bias module.network_current_names = wanted_names if task is not None: - pbar.update(task, advance=1, description=f'networks={len(loaded_networks)} modules={total} apply={len(applied_layers)} backup={backup_size}') + pbar.update(task, advance=1, description=f'networks={len(loaded_networks)} modules={active_components} layers={total} apply={len(applied_layers)} backup={backup_size}') + if task is not None and len(applied_layers) == 0: pbar.remove_task(task) # hide progress bar for no action - weights_devices, weights_dtypes = list(set([x for x in weights_devices if x is not None])), list(set([x for x in weights_dtypes if x is not None])) # noqa: C403 # pylint: disable=R1718 timer.activate += time.time() - t0 if debug and len(loaded_networks) > 0: - shared.log.debug(f'Load network: type=LoRA networks={len(loaded_networks)} components={components} modules={total} apply={len(applied_layers)} device={weights_devices} dtype={weights_dtypes} backup={backup_size} fuse={shared.opts.lora_fuse_diffusers} time={timer.summary}') + weights_devices, weights_dtypes = list(set([x for x in weights_devices if x is not None])), list(set([x for x in weights_dtypes if x is not None])) # noqa: C403 # pylint: disable=R1718 + shared.log.debug(f'Load network: type=LoRA networks={[n.name for n in loaded_networks]} modules={active_components} layers={total} apply={len(applied_layers)} device={weights_devices} dtype={weights_dtypes} backup={backup_size} fuse={shared.opts.lora_fuse_diffusers} time={timer.summary}') modules.clear() if shared.opts.diffusers_offload_mode == "sequential": sd_models.set_diffuser_offload(sd_model, op="model") diff --git a/modules/model_flux.py b/modules/model_flux.py index dd0a507db..4ba1bb556 100644 --- a/modules/model_flux.py +++ b/modules/model_flux.py @@ -148,9 +148,10 @@ def load_quants(kwargs, repo_id, cache_dir): quant_args = model_quant.create_bnb_config(quant_args) if quant_args: model_quant.load_bnb(f'Load model: type=FLUX quant={quant_args}') - quant_args = model_quant.create_ao_config(quant_args) - if quant_args: - model_quant.load_torchao(f'Load model: type=FLUX quant={quant_args}') + if not quant_args: + quant_args = model_quant.create_ao_config(quant_args) + if quant_args: + model_quant.load_torchao(f'Load model: type=FLUX quant={quant_args}') if not quant_args: return kwargs if 'transformer' not in kwargs and ('Model' in shared.opts.bnb_quantization or 'Model' in shared.opts.torchao_quantization): diff --git a/modules/model_sana.py b/modules/model_sana.py index 792b49d15..7dc551a6f 100644 --- a/modules/model_sana.py +++ b/modules/model_sana.py @@ -11,9 +11,10 @@ def load_quants(kwargs, repo_id, cache_dir): quant_args = model_quant.create_bnb_config(quant_args) if quant_args: model_quant.load_bnb(f'Load model: type=Sana quant={quant_args}') - quant_args = model_quant.create_ao_config(quant_args) - if quant_args: - model_quant.load_torchao(f'Load model: type=Sana quant={quant_args}') + if not quant_args: + quant_args = model_quant.create_ao_config(quant_args) + if quant_args: + model_quant.load_torchao(f'Load model: type=Sana quant={quant_args}') if not quant_args: return kwargs load_args = kwargs.copy() diff --git a/modules/model_sd3.py b/modules/model_sd3.py index df9a5f780..d0e23026b 100644 --- a/modules/model_sd3.py +++ b/modules/model_sd3.py @@ -56,9 +56,10 @@ def load_quants(kwargs, repo_id, cache_dir): quant_args = model_quant.create_bnb_config(quant_args) if quant_args: model_quant.load_bnb(f'Load model: type=SD3 quant={quant_args}') - quant_args = model_quant.create_ao_config(quant_args) - if quant_args: - model_quant.load_torchao(f'Load model: type=SD3 quant={quant_args}') + if not quant_args: + quant_args = model_quant.create_ao_config(quant_args) + if quant_args: + model_quant.load_torchao(f'Load model: type=SD3 quant={quant_args}') if not quant_args: return kwargs if 'Model' in shared.opts.bnb_quantization and 'transformer' not in kwargs: diff --git a/modules/processing_args.py b/modules/processing_args.py index c744c3d14..054211517 100644 --- a/modules/processing_args.py +++ b/modules/processing_args.py @@ -221,7 +221,10 @@ def set_pipeline_args(p, model, prompts: list, negative_prompts: list, prompts_2 if 'img_guidance_scale' in possible and hasattr(p, 'image_cfg_scale'): args['img_guidance_scale'] = p.image_cfg_scale if 'generator' in possible: - args['generator'] = get_generator(p) + generator = get_generator(p) + args['generator'] = generator + else: + generator = None if 'latents' in possible and getattr(p, "init_latent", None) is not None: if sd_models.get_diffusers_task(model) == sd_models.DiffusersTaskType.TEXT_2_IMAGE: args['latents'] = p.init_latent @@ -321,7 +324,8 @@ def set_pipeline_args(p, model, prompts: list, negative_prompts: list, prompts_2 clean['prompt'] = len(clean['prompt']) if 'negative_prompt' in clean and clean['negative_prompt'] is not None: clean['negative_prompt'] = len(clean['negative_prompt']) - clean.pop('generator', None) + if generator is not None: + clean['generator'] = f'{generator[0].device}:{[g.initial_seed() for g in generator]}' clean['parser'] = parser for k, v in clean.copy().items(): if isinstance(v, torch.Tensor) or isinstance(v, np.ndarray): @@ -331,7 +335,7 @@ def set_pipeline_args(p, model, prompts: list, negative_prompts: list, prompts_2 if not debug_enabled and k.endswith('_embeds'): del clean[k] clean['prompt'] = 'embeds' - shared.log.debug(f'Diffuser pipeline: {model.__class__.__name__} task={sd_models.get_diffusers_task(model)} batch={p.iteration + 1}/{p.n_iter}x{p.batch_size} set={clean}') + shared.log.info(f'{desc}: pipeline={model.__class__.__name__} task={sd_models.get_diffusers_task(model)} batch={p.iteration + 1}/{p.n_iter}x{p.batch_size} set={clean}') if p.hdr_clamp or p.hdr_maximize or p.hdr_brightness != 0 or p.hdr_color != 0 or p.hdr_sharpen != 0: txt = 'HDR:' diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index a108a1095..41d412a68 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -57,7 +57,6 @@ def process_base(p: processing.StableDiffusionProcessing): use_denoise_start = not is_txt2img() and p.refiner_start > 0 and p.refiner_start < 1 shared.sd_model = update_pipeline(shared.sd_model, p) - shared.log.info(f'Base: class={shared.sd_model.__class__.__name__}') update_sampler(p, shared.sd_model) timer.process.record('prepare') base_args = set_pipeline_args( @@ -90,7 +89,7 @@ def process_base(p: processing.StableDiffusionProcessing): sd_models.move_model(shared.sd_model.unet, devices.device) if hasattr(shared.sd_model, 'transformer'): sd_models.move_model(shared.sd_model.transformer, devices.device) - extra_networks.activate(p, exclude=['text_encoder', 'text_encoder_2']) + extra_networks.activate(p, exclude=['text_encoder', 'text_encoder_2', 'text_encoder_3']) hidiffusion.apply(p, shared.sd_model_type) # if 'image' in base_args: # base_args['image'] = set_latents(p) @@ -195,7 +194,6 @@ def process_hires(p: processing.StableDiffusionProcessing, output): if p.hr_force: shared.state.job_count = 2 * p.n_iter shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE) - shared.log.info(f'HiRes: class={shared.sd_model.__class__.__name__} sampler="{p.hr_sampler_name}"') if 'Upscale' in shared.sd_model.__class__.__name__ or 'Flux' in shared.sd_model.__class__.__name__: output.images = processing_vae.vae_decode(latents=output.images, model=shared.sd_model, full_quality=p.full_quality, output_type='pil', width=p.width, height=p.height) if p.is_control and hasattr(p, 'task_args') and p.task_args.get('image', None) is not None: @@ -294,7 +292,6 @@ def process_refine(p: processing.StableDiffusionProcessing, output): if p.task_args.get('image', None) is not None and output is not None: # replace input with output so it can be used by hires/refine # p.task_args['image'] = image p.init_images = [image] - shared.log.info(f'Refiner: class={shared.sd_refiner.__class__.__name__}') update_sampler(p, shared.sd_refiner, second_pass=True) refiner_args = set_pipeline_args( p=p, diff --git a/modules/processing_helpers.py b/modules/processing_helpers.py index 304e2c211..51cbcff7f 100644 --- a/modules/processing_helpers.py +++ b/modules/processing_helpers.py @@ -511,10 +511,9 @@ def get_generator(p): else: generator_device = devices.cpu if shared.opts.diffusers_generator_device == "CPU" else shared.device try: + p.seeds = [seed if seed != -1 else get_fixed_seed(seed) for seed in p.seeds if seed] devices.randn(p.seeds[0]) generator = [torch.Generator(generator_device).manual_seed(s) for s in p.seeds] - seeds = [g.initial_seed() for g in generator] - shared.log.debug(f'Torch generator: device={generator_device} seeds={seeds}') except Exception as e: shared.log.error(f'Torch generator: seeds={p.seeds} device={generator_device} {e}') generator = None diff --git a/modules/processing_vae.py b/modules/processing_vae.py index 95b83daa4..034d17065 100644 --- a/modules/processing_vae.py +++ b/modules/processing_vae.py @@ -135,9 +135,9 @@ def full_vae_decode(latents, model): latents = latents + shift_factor vae_name = os.path.splitext(os.path.basename(sd_vae.loaded_vae_file))[0] if sd_vae.loaded_vae_file is not None else "default" - vae_stats = f'name="{vae_name}" dtype={model.vae.dtype} device={model.vae.device} upcast={upcast} slicing={getattr(model.vae, "use_slicing", None)} tiling={getattr(model.vae, "use_tiling", None)}' + vae_stats = f'vae="{vae_name}" dtype={model.vae.dtype} device={model.vae.device} upcast={upcast} slicing={getattr(model.vae, "use_slicing", None)} tiling={getattr(model.vae, "use_tiling", None)}' latents_stats = f'shape={latents.shape} dtype={latents.dtype} device={latents.device}' - stats = f'vae {vae_stats} latents {latents_stats}' + stats = f'{vae_stats} latents {latents_stats}' log_debug(f'VAE config: {model.vae.config}') try: @@ -165,7 +165,7 @@ def full_vae_decode(latents, model): t1 = time.time() if debug: log_debug(f'VAE memory: {shared.mem_mon.read()}') - shared.log.debug(f'VAE decode: {stats} time={round(t1-t0, 3)}') + shared.log.debug(f'Decode: {stats} time={round(t1-t0, 3)}') return decoded diff --git a/modules/sd_models.py b/modules/sd_models.py index f3d0f569f..99e821667 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -510,7 +510,7 @@ def apply_balanced_offload_to_module(pipe): module = module.to(devices.cpu, non_blocking=True) used_gpu -= module_size if not cached: - shared.log.debug(f'Offload: type=balanced module={module_name} cls={module.__class__.__name__} dtype={module.dtype} quant={getattr(module, "quantization_method", None)} params={offload_hook_instance.param_map[module_name]:.3f} size={offload_hook_instance.offload_map[module_name]:.3f}') + shared.log.debug(f'Model module={module_name} type={module.__class__.__name__} dtype={module.dtype} quant={getattr(module, "quantization_method", None)} params={offload_hook_instance.param_map[module_name]:.3f} size={offload_hook_instance.offload_map[module_name]:.3f}') debug_move(f'Offload: type=balanced op={"move" if do_offload else "skip"} gpu={prev_gpu:.3f}:{used_gpu:.3f} perc={perc_gpu:.2f} ram={used_ram:.3f} current={module.device} dtype={module.dtype} quant={getattr(module, "quantization_method", None)} module={module.__class__.__name__} size={module_size:.3f}') except Exception as e: if 'out of memory' in str(e): @@ -544,7 +544,7 @@ def apply_balanced_offload_to_module(pipe): fn = f'{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access debug_move(f'Apply offload: time={t:.2f} type=balanced fn={fn}') if not cached: - shared.log.info(f'Offload: type=balanced op=apply class={sd_model.__class__.__name__} modules={len(offload_hook_instance.offload_map)} size={offload_hook_instance.model_size():.3f}') + shared.log.info(f'Model class={sd_model.__class__.__name__} modules={len(offload_hook_instance.offload_map)} size={offload_hook_instance.model_size():.3f}') return sd_model diff --git a/modules/sd_samplers.py b/modules/sd_samplers.py index 5159785c4..252e52d0f 100644 --- a/modules/sd_samplers.py +++ b/modules/sd_samplers.py @@ -61,7 +61,7 @@ def create_sampler(name, model): model.prior_pipe.scheduler = copy.deepcopy(model.default_scheduler) model.prior_pipe.scheduler.config.clip_sample = False config = {k: v for k, v in model.scheduler.config.items() if not k.startswith('_')} - shared.log.debug(f'Sampler: sampler=default class={current}: {config}') + shared.log.debug(f'Sampler: default class={current}: {config}') if "flow" in model.scheduler.__class__.__name__.lower(): shared.state.prediction_type = "flow_prediction" elif hasattr(model.scheduler, "config") and hasattr(model.scheduler.config, "prediction_type"): @@ -77,7 +77,7 @@ def create_sampler(name, model): sampler.config = config sampler.name = name sampler.initialize(p=None) - shared.log.debug(f'Sampler: sampler="{name}" config={config.options}') + shared.log.debug(f'Sampler: "{name}" config={config.options}') return sampler elif shared.native: FlowModels = ['Flux', 'StableDiffusion3', 'Lumina', 'AuraFlow', 'Sana', 'HunyuanVideoPipeline'] @@ -103,7 +103,7 @@ def create_sampler(name, model): elif hasattr(model.scheduler, "config") and hasattr(model.scheduler.config, "prediction_type"): shared.state.prediction_type = model.scheduler.config.prediction_type clean_config = {k: v for k, v in sampler.config.items() if v is not None and v is not False} - shared.log.debug(f'Sampler: sampler="{sampler.name}" class="{model.scheduler.__class__.__name__} config={clean_config}') + shared.log.debug(f'Sampler: "{sampler.name}" class={model.scheduler.__class__.__name__} config={clean_config}') return sampler.sampler else: return None diff --git a/modules/styles.py b/modules/styles.py index d0228d33a..85ae190e2 100644 --- a/modules/styles.py +++ b/modules/styles.py @@ -103,9 +103,9 @@ def apply_wildcards_to_prompt(prompt, all_wildcards, seed=-1, silent=False): prompt, replaced_file, not_found = apply_file_wildcards(prompt, [], [], recursion=0, seed=seed) t2 = time.time() if replaced and not silent: - shared.log.debug(f'Wildcards applied: {replaced} path="{shared.opts.wildcards_dir}" type=style time={t1-t0:.2f}') + shared.log.debug(f'Apply wildcards: {replaced} path="{shared.opts.wildcards_dir}" type=style time={t1-t0:.2f}') if (len(replaced_file) > 0 or len(not_found) > 0) and not silent: - shared.log.debug(f'Wildcards applied: {replaced_file} missing: {not_found} path="{shared.opts.wildcards_dir}" type=file time={t2-t2:.2f} ') + shared.log.debug(f'Apply wildcards: {replaced_file} missing: {not_found} path="{shared.opts.wildcards_dir}" type=file time={t2-t2:.2f} ') if old_state is not None: random.setstate(old_state) return prompt @@ -158,7 +158,7 @@ def apply_styles_to_extra(p, style: Style): fields.append(f'{k}={v}') else: skipped.append(f'{k}={v}') - shared.log.debug(f'Applying style: name="{style.name}" extra={fields} skipped={skipped} reference={True if reference_style else False}') + shared.log.debug(f'Apply style: name="{style.name}" extra={fields} skipped={skipped} reference={True if reference_style else False}') class StyleDatabase: diff --git a/scripts/hunyuanvideo.py b/scripts/hunyuanvideo.py index d4ceee53e..0e99a26a9 100644 --- a/scripts/hunyuanvideo.py +++ b/scripts/hunyuanvideo.py @@ -104,9 +104,10 @@ def run(self, p: processing.StableDiffusionProcessing, num_frames, tile_frames, quant_args = model_quant.create_bnb_config(quant_args) if quant_args: model_quant.load_bnb(f'Load model: type=HunyuanVideo quant={quant_args}') - quant_args = model_quant.create_ao_config(quant_args) - if quant_args: - model_quant.load_torchao(f'Load model: type=HunyuanVideo quant={quant_args}') + if not quant_args: + quant_args = model_quant.create_ao_config(quant_args) + if quant_args: + model_quant.load_torchao(f'Load model: type=HunyuanVideo quant={quant_args}') transformer = diffusers.HunyuanVideoTransformer3DModel.from_pretrained( repo_id, subfolder="transformer", diff --git a/scripts/ltxvideo.py b/scripts/ltxvideo.py index 279d98d92..d4f62bc2b 100644 --- a/scripts/ltxvideo.py +++ b/scripts/ltxvideo.py @@ -19,9 +19,10 @@ def load_quants(kwargs, repo_id): quant_args = model_quant.create_bnb_config(quant_args) if quant_args: model_quant.load_bnb(f'Load model: type=LTXVideo quant={quant_args}') - quant_args = model_quant.create_ao_config(quant_args) - if quant_args: - model_quant.load_torchao(f'Load model: type=LTXVideo quant={quant_args}') + if not quant_args: + quant_args = model_quant.create_ao_config(quant_args) + if quant_args: + model_quant.load_torchao(f'Load model: type=LTXVideo quant={quant_args}') if not quant_args: return kwargs model_quant.load_bnb(f'Load model: type=LTX quant={quant_args}') From 0d5c8444f9a41395deb880864a1cd2ae201017f4 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 29 Dec 2024 12:42:08 -0500 Subject: [PATCH 170/249] refactor lora add_weights Signed-off-by: Vladimir Mandic --- modules/lora/networks.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/modules/lora/networks.py b/modules/lora/networks.py index 7863e943b..0be0a25f7 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -411,26 +411,26 @@ def network_calc_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn. return batch_updown, batch_ex_bias -def add_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv], weights: Union[None, torch.Tensor] = None, bias: torch.Tensor = None, deactivate: bool = False): - if bias is None: +def network_add_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn.GroupNorm, torch.nn.LayerNorm, diffusers.models.lora.LoRACompatibleLinear, diffusers.models.lora.LoRACompatibleConv], model_weights: Union[None, torch.Tensor] = None, lora_weights: torch.Tensor = None, deactivate: bool = False): + if lora_weights is None: return self.weight if deactivate: - bias *= -1 - if weights is None: # weights are used if provided-from-backup else use self.weight - weights = self.weight + lora_weights *= -1 + if model_weights is None: # weights are used if provided-from-backup else use self.weight + model_weights = self.weight # TODO lora: add other quantization types if self.__class__.__name__ == 'Linear4bit' and bnb is not None: try: - dequant_weight = bnb.functional.dequantize_4bit(weights.to(devices.device), quant_state=self.quant_state, quant_type=self.quant_type, blocksize=self.blocksize) - new_weight = dequant_weight.to(devices.device) + bias.to(devices.device) + dequant_weight = bnb.functional.dequantize_4bit(model_weights.to(devices.device), quant_state=self.quant_state, quant_type=self.quant_type, blocksize=self.blocksize) + new_weight = dequant_weight.to(devices.device) + lora_weights.to(devices.device) self.weight = bnb.nn.Params4bit(new_weight, quant_state=self.quant_state, quant_type=self.quant_type, blocksize=self.blocksize) except Exception as e: - shared.log.error(f'Load network: type=LoRA quant=bnb cls={self.__class__.__name__} type={self.quant_type} blocksize={self.blocksize} state={vars(self.quant_state)} weight={self.weight} bias={bias} {e}') + shared.log.error(f'Load network: type=LoRA quant=bnb cls={self.__class__.__name__} type={self.quant_type} blocksize={self.blocksize} state={vars(self.quant_state)} weight={self.weight} bias={lora_weights} {e}') else: try: - new_weight = weights.to(devices.device) + bias.to(devices.device) + new_weight = model_weights.to(devices.device) + lora_weights.to(devices.device) except Exception: - new_weight = weights + bias # try without device cast + new_weight = model_weights + lora_weights # try without device cast self.weight = torch.nn.Parameter(new_weight, requires_grad=False) try: self.weight = self.weight.to(device=devices.device) # required since quantization happens only during .to call, not during params creation @@ -454,11 +454,11 @@ def network_apply_direct(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn. if updown is not None and len(self.weight.shape) == 4 and self.weight.shape[1] == 9: # inpainting model. zero pad updown to make channel[1] 4 to 9 updown = torch.nn.functional.pad(updown, (0, 0, 0, 0, 0, 5)) # pylint: disable=not-callable if updown is not None: - self.weight = add_weights(self, bias=updown, deactivate=deactivate) + self.weight = network_add_weights(self, lora_weights=updown, deactivate=deactivate) if bias_backup: if ex_bias is not None: - self.bias = add_weights(self, bias=ex_bias, deactivate=deactivate) + self.bias = network_add_weights(self, lora_weights=ex_bias, deactivate=deactivate) if hasattr(self, "qweight") and hasattr(self, "freeze"): self.freeze() @@ -479,14 +479,14 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn if updown is not None and len(weights_backup.shape) == 4 and weights_backup.shape[1] == 9: # inpainting model. zero pad updown to make channel[1] 4 to 9 updown = torch.nn.functional.pad(updown, (0, 0, 0, 0, 0, 5)) # pylint: disable=not-callable if updown is not None: - self.weight = add_weights(self, weights=weights_backup, bias=updown, deactivate=deactivate) + self.weight = network_add_weights(self, model_weights=weights_backup, lora_weights=updown, deactivate=deactivate) else: self.weight = torch.nn.Parameter(weights_backup.to(device=orig_device), requires_grad=False) if bias_backup is not None: self.bias = None if ex_bias is not None: - self.weight = add_weights(self, weights=weights_backup, bias=ex_bias, deactivate=deactivate) + self.weight = network_add_weights(self, model_weights=weights_backup, lora_weights=ex_bias, deactivate=deactivate) else: self.bias = torch.nn.Parameter(bias_backup.to(device=orig_device), requires_grad=False) From 6ea4d713e4a6a6cd4541b3ed79699b434b73b694 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 29 Dec 2024 13:54:14 -0500 Subject: [PATCH 171/249] live preview error handling Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 3 +-- extensions-builtin/Lora/ui_extra_networks_lora.py | 1 - javascript/progressBar.js | 2 +- modules/model_flux.py | 2 +- modules/model_sana.py | 13 ++++++------- modules/model_sd3.py | 2 +- modules/sd_samplers_common.py | 2 ++ modules/sd_vae_taesd.py | 5 +++-- modules/shared_state.py | 11 +++++++---- scripts/ltxvideo.py | 4 ++-- 10 files changed, 24 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe2edffe8..a7b001187 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,12 +10,11 @@ - **HunyuanVideo** optimizations: full offload, quantization and tiling support - **LTXVideo** optimizations: full offload, quantization and tiling support - VAE tiling granular options in *settings -> variable auto encoder* -- UI: live preview add sigma calculation, thanks @Disty0 +- UI: live preview optimizations - UI: CSS optimizations when log view is disabled - Samplers: add flow shift options and separate dynamic thresholding from dynamic shifting - **Fixes** - do not show disabled networks - - live preview image sizes in modern and standard UI - image width/height calculation when doing img2img - flux pipeline switches: txt/img/inpaint - interrogate caption with T5 diff --git a/extensions-builtin/Lora/ui_extra_networks_lora.py b/extensions-builtin/Lora/ui_extra_networks_lora.py index c5adbb81f..cd7875377 100644 --- a/extensions-builtin/Lora/ui_extra_networks_lora.py +++ b/extensions-builtin/Lora/ui_extra_networks_lora.py @@ -80,7 +80,6 @@ def find_version(): def create_item(self, name): l = networks.available_networks.get(name) if l is None: - print('HERE1') shared.log.warning(f'Networks: type=lora registered={len(list(networks.available_networks))} file="{name}" not registered') return None try: diff --git a/javascript/progressBar.js b/javascript/progressBar.js index dfd895f8e..afb9259dc 100644 --- a/javascript/progressBar.js +++ b/javascript/progressBar.js @@ -118,7 +118,7 @@ function requestProgress(id_task, progressEl, galleryEl, atEnd = null, onProgres const elapsedFromStart = (new Date() - dateStart) / 1000; hasStarted |= res.active; if (res.completed || (!res.active && (hasStarted || once)) || (elapsedFromStart > 30 && !res.queued && res.progress === prevProgress)) { - debug('onProgressEnd', res); + // debug('onProgressEnd', res); done(); return; } diff --git a/modules/model_flux.py b/modules/model_flux.py index 4ba1bb556..7a3fa9e00 100644 --- a/modules/model_flux.py +++ b/modules/model_flux.py @@ -157,7 +157,7 @@ def load_quants(kwargs, repo_id, cache_dir): if 'transformer' not in kwargs and ('Model' in shared.opts.bnb_quantization or 'Model' in shared.opts.torchao_quantization): kwargs['transformer'] = diffusers.FluxTransformer2DModel.from_pretrained(repo_id, subfolder="transformer", cache_dir=cache_dir, torch_dtype=devices.dtype, **quant_args) shared.log.debug(f'Quantization: module=transformer type=bnb dtype={shared.opts.bnb_quantization_type} storage={shared.opts.bnb_quantization_storage}') - if 'text_encoder_3' not in kwargs and ('Text Encoder' in shared.opts.bnb_quantization or 'Text Encoder' in shared.opts.torchao_quantization): + if 'text_encoder_2' not in kwargs and ('Text Encoder' in shared.opts.bnb_quantization or 'Text Encoder' in shared.opts.torchao_quantization): kwargs['text_encoder_2'] = transformers.T5EncoderModel.from_pretrained(repo_id, subfolder="text_encoder_2", cache_dir=cache_dir, torch_dtype=devices.dtype, **quant_args) shared.log.debug(f'Quantization: module=t5 type=bnb dtype={shared.opts.bnb_quantization_type} storage={shared.opts.bnb_quantization_storage}') return kwargs diff --git a/modules/model_sana.py b/modules/model_sana.py index 7dc551a6f..79a13592d 100644 --- a/modules/model_sana.py +++ b/modules/model_sana.py @@ -18,12 +18,11 @@ def load_quants(kwargs, repo_id, cache_dir): if not quant_args: return kwargs load_args = kwargs.copy() - model_quant.load_bnb(f'Load model: type=SD3 quant={quant_args} args={load_args}') - if 'Model' in shared.opts.bnb_quantization and 'transformer' not in kwargs: + if 'transformer' not in kwargs and ('Model' in shared.opts.bnb_quantization or 'Model' in shared.opts.torchao_quantization): kwargs['transformer'] = diffusers.models.SanaTransformer2DModel.from_pretrained(repo_id, subfolder="transformer", cache_dir=cache_dir, **load_args, **quant_args) shared.log.debug(f'Quantization: module=transformer type=bnb dtype={shared.opts.bnb_quantization_type} storage={shared.opts.bnb_quantization_storage}') - if 'Text Encoder' in shared.opts.bnb_quantization and 'text_encoder_3' not in kwargs: - kwargs['text_encoder_3'] = transformers.AutoModelForCausalLM.from_pretrained(repo_id, subfolder="text_encoder", cache_dir=cache_dir, **load_args, **quant_args) + if 'text_encoder' not in kwargs and ('Text Encoder' in shared.opts.bnb_quantization or 'Text Encoder' in shared.opts.torchao_quantization): + kwargs['text_encoder'] = transformers.AutoModelForCausalLM.from_pretrained(repo_id, subfolder="text_encoder", cache_dir=cache_dir, **load_args, **quant_args) shared.log.debug(f'Quantization: module=t5 type=bnb dtype={shared.opts.bnb_quantization_type} storage={shared.opts.bnb_quantization_storage}') return kwargs @@ -55,9 +54,9 @@ def load_sana(checkpoint_info, kwargs={}): kwargs['variant'] = 'fp16' if (fn is None) or (not os.path.exists(fn) or os.path.isdir(fn)): - kwargs = load_quants(kwargs, repo_id, cache_dir=shared.opts.diffusers_dir) - # kwargs = model_quant.create_bnb_config(kwargs) - # kwargs = model_quant.create_ao_config(kwargs) + # TODO sana: fails when quantized + # kwargs = load_quants(kwargs, repo_id, cache_dir=shared.opts.diffusers_dir) + pass shared.log.debug(f'Load model: type=Sana repo="{repo_id}" args={list(kwargs)}') t0 = time.time() pipe = diffusers.SanaPipeline.from_pretrained(repo_id, cache_dir=shared.opts.diffusers_dir, **kwargs) diff --git a/modules/model_sd3.py b/modules/model_sd3.py index d0e23026b..bf8644284 100644 --- a/modules/model_sd3.py +++ b/modules/model_sd3.py @@ -65,7 +65,7 @@ def load_quants(kwargs, repo_id, cache_dir): if 'Model' in shared.opts.bnb_quantization and 'transformer' not in kwargs: kwargs['transformer'] = diffusers.SD3Transformer2DModel.from_pretrained(repo_id, subfolder="transformer", cache_dir=cache_dir, torch_dtype=devices.dtype, **quant_args) shared.log.debug(f'Quantization: module=transformer type=bnb dtype={shared.opts.bnb_quantization_type} storage={shared.opts.bnb_quantization_storage}') - if 'Text Encoder' in shared.opts.bnb_quantization and 'text_encoder_3' not in kwargs: + if 'text_encoder_3' not in kwargs and ('Text Encoder' in shared.opts.bnb_quantization or 'Text Encoder' in shared.opts.torchao_quantization): kwargs['text_encoder_3'] = transformers.T5EncoderModel.from_pretrained(repo_id, subfolder="text_encoder_3", variant='fp16', cache_dir=cache_dir, torch_dtype=devices.dtype, **quant_args) shared.log.debug(f'Quantization: module=t5 type=bnb dtype={shared.opts.bnb_quantization_type} storage={shared.opts.bnb_quantization_storage}') return kwargs diff --git a/modules/sd_samplers_common.py b/modules/sd_samplers_common.py index a90ceec27..c4a2712f3 100644 --- a/modules/sd_samplers_common.py +++ b/modules/sd_samplers_common.py @@ -71,6 +71,8 @@ def single_sample_to_image(sample, approximation=None): warn_once(f"Unknown latent decode type: {approximation}") return Image.new(mode="RGB", size=(512, 512)) try: + if x_sample.shape[0] > 4: + return Image.new(mode="RGB", size=(512, 512)) if x_sample.dtype == torch.bfloat16: x_sample.to(torch.float16) transform = T.ToPILImage() diff --git a/modules/sd_vae_taesd.py b/modules/sd_vae_taesd.py index a1959817c..4507ee3c8 100644 --- a/modules/sd_vae_taesd.py +++ b/modules/sd_vae_taesd.py @@ -153,8 +153,9 @@ def decode(latents): if not previous_warnings: previous_warnings = True shared.log.warning(f'TAESD unsupported model type: {model_class}') - return Image.new('RGB', (8, 8), color = (0, 0, 0)) - vae = taesd_models[f'{model_class}-decoder'] + # return Image.new('RGB', (8, 8), color = (0, 0, 0)) + return latents + vae = taesd_models.get(f'{model_class}-decoder', None) if vae is None: model_path = os.path.join(paths.models_path, "TAESD", f"tae{model_class}_decoder.pth") download_model(model_path) diff --git a/modules/shared_state.py b/modules/shared_state.py index c773b36a9..024427f09 100644 --- a/modules/shared_state.py +++ b/modules/shared_state.py @@ -1,7 +1,7 @@ import os import time import datetime -from modules.errors import log +from modules.errors import log, display class State: @@ -164,17 +164,20 @@ def do_set_current_image(self): try: sample = self.current_latent self.current_image_sampling_step = self.sampling_step + """ if self.current_noise_pred is not None and self.current_sigma is not None and self.current_sigma_next is not None: original_sample = sample - (self.current_noise_pred * (self.current_sigma_next-self.current_sigma)) + # RuntimeError: The size of tensor a (128) must match the size of tensor b (64) at non-singleton dimension 3 if self.prediction_type in {"epsilon", "flow_prediction"}: sample = original_sample - (self.current_noise_pred * self.current_sigma) elif self.prediction_type == "v_prediction": sample = self.current_noise_pred * (-self.current_sigma / (self.current_sigma**2 + 1) ** 0.5) + (original_sample / (self.current_sigma**2 + 1)) # pylint: disable=invalid-unary-operand-type + """ image = sd_samplers.samples_to_image_grid(sample) if shared.opts.show_progress_grid else sd_samplers.sample_to_image(sample) self.assign_current_image(image) - except Exception: - # log.error(f'Error setting current image: step={self.sampling_step} {e}') - pass + except Exception as e: + log.error(f'State image: last={self.id_live_preview} step={self.sampling_step} {e}') + display(e, 'State image') self.preview_busy = False def assign_current_image(self, image): diff --git a/scripts/ltxvideo.py b/scripts/ltxvideo.py index d4f62bc2b..7b12a3a86 100644 --- a/scripts/ltxvideo.py +++ b/scripts/ltxvideo.py @@ -26,10 +26,10 @@ def load_quants(kwargs, repo_id): if not quant_args: return kwargs model_quant.load_bnb(f'Load model: type=LTX quant={quant_args}') - if 'Model' in shared.opts.bnb_quantization and 'transformer' not in kwargs: + if 'transformer' not in kwargs and ('Model' in shared.opts.bnb_quantization or 'Model' in shared.opts.torchao_quantization): kwargs['transformer'] = diffusers.LTXVideoTransformer3DModel.from_pretrained(repo_id, subfolder="transformer", cache_dir=shared.opts.hfcache_dir, torch_dtype=devices.dtype, **quant_args) shared.log.debug(f'Quantization: module=transformer type=bnb dtype={shared.opts.bnb_quantization_type} storage={shared.opts.bnb_quantization_storage}') - if 'Text Encoder' in shared.opts.bnb_quantization and 'text_encoder_3' not in kwargs: + if 'text_encoder' not in kwargs and ('Text Encoder' in shared.opts.bnb_quantization or 'Text Encoder' in shared.opts.torchao_quantization): kwargs['text_encoder'] = transformers.T5EncoderModel.from_pretrained(repo_id, subfolder="text_encoder", cache_dir=shared.opts.hfcache_dir, torch_dtype=devices.dtype, **quant_args) shared.log.debug(f'Quantization: module=t5 type=bnb dtype={shared.opts.bnb_quantization_type} storage={shared.opts.bnb_quantization_storage}') return kwargs From 14bf2dab66f4ccd470eeb762fd86bd9d8de023e6 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 29 Dec 2024 15:41:34 -0500 Subject: [PATCH 172/249] typo Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 2 +- modules/lora/extra_networks_lora.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a7b001187..df9a98edd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ - **HunyuanVideo** optimizations: full offload, quantization and tiling support - **LTXVideo** optimizations: full offload, quantization and tiling support - VAE tiling granular options in *settings -> variable auto encoder* -- UI: live preview optimizations +- UI: live preview optimizations and error handling - UI: CSS optimizations when log view is disabled - Samplers: add flow shift options and separate dynamic thresholding from dynamic shifting - **Fixes** diff --git a/modules/lora/extra_networks_lora.py b/modules/lora/extra_networks_lora.py index 9708fb170..c21264896 100644 --- a/modules/lora/extra_networks_lora.py +++ b/modules/lora/extra_networks_lora.py @@ -127,7 +127,7 @@ def changed(self, requested: List[str], include: List[str], exclude: List[str]): sd_model = getattr(shared.sd_model, "pipe", shared.sd_model) if not hasattr(sd_model, 'loaded_loras'): sd_model.loaded_loras = {} - key = f'{','.join(include)}:{','.join(exclude)}' + key = f'{",".join(include)}:{','.join(exclude)}' loaded = sd_model.loaded_loras.get(key, []) # shared.log.trace(f'Load network: type=LoRA key="{key}" requested={requested} loaded={loaded}') if len(requested) != len(loaded): From b3ae6d461958a87d6118cabee37311fe14f232d9 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 29 Dec 2024 15:42:29 -0500 Subject: [PATCH 173/249] again Signed-off-by: Vladimir Mandic --- modules/lora/extra_networks_lora.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/lora/extra_networks_lora.py b/modules/lora/extra_networks_lora.py index c21264896..d9b8f81f2 100644 --- a/modules/lora/extra_networks_lora.py +++ b/modules/lora/extra_networks_lora.py @@ -127,7 +127,7 @@ def changed(self, requested: List[str], include: List[str], exclude: List[str]): sd_model = getattr(shared.sd_model, "pipe", shared.sd_model) if not hasattr(sd_model, 'loaded_loras'): sd_model.loaded_loras = {} - key = f'{",".join(include)}:{','.join(exclude)}' + key = f'{",".join(include)}:{",".join(exclude)}' loaded = sd_model.loaded_loras.get(key, []) # shared.log.trace(f'Load network: type=LoRA key="{key}" requested={requested} loaded={loaded}') if len(requested) != len(loaded): From c093e07845ddc39eb0c57f1952a0ecb2e4cab9bd Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 29 Dec 2024 17:01:14 -0500 Subject: [PATCH 174/249] livepreview fixes and debug Signed-off-by: Vladimir Mandic --- javascript/progressBar.js | 16 +++++++++++----- modules/progress.py | 17 +++++++++++------ modules/shared_state.py | 20 ++++++++++++-------- 3 files changed, 34 insertions(+), 19 deletions(-) diff --git a/javascript/progressBar.js b/javascript/progressBar.js index afb9259dc..3f5c9fcef 100644 --- a/javascript/progressBar.js +++ b/javascript/progressBar.js @@ -61,8 +61,8 @@ function randomId() { function requestProgress(id_task, progressEl, galleryEl, atEnd = null, onProgress = null, once = false) { localStorage.setItem('task', id_task); let hasStarted = false; - const dateStart = new Date(); - const prevProgress = null; + let dateStart = new Date(); + let prevProgress = null; const parentGallery = galleryEl ? galleryEl.parentNode : null; let livePreview; let img; @@ -113,30 +113,36 @@ function requestProgress(id_task, progressEl, galleryEl, atEnd = null, onProgres if (!opts.live_previews_enable || opts.live_preview_refresh_period === 0 || opts.show_progress_every_n_steps === 0) return; const onProgressHandler = (res) => { - // debug('onProgress', res); + if (res?.debug) debug('livePreview:', dateStart, res); lastState = res; const elapsedFromStart = (new Date() - dateStart) / 1000; hasStarted |= res.active; if (res.completed || (!res.active && (hasStarted || once)) || (elapsedFromStart > 30 && !res.queued && res.progress === prevProgress)) { - // debug('onProgressEnd', res); + if (res?.debug) debug('livePreview end:', res); done(); return; } + if (res.progress !== prevProgress) { + dateStart = new Date(); + prevProgress = res.progress; + } setProgress(res); if (res.live_preview && !livePreview) initLivePreview(); if (res.live_preview && galleryEl) { if (img.src !== res.live_preview) img.src = res.live_preview; + id_live_preview = res.id_live_preview; } if (onProgress) onProgress(res); setTimeout(() => start(id_task, id_live_preview), opts.live_preview_refresh_period || 500); }; const onProgressErrorHandler = (err) => { - error(`onProgressError: ${err}`); + error(`livePreview: ${err}`); done(); }; xhrPost('./internal/progress', { id_task, id_live_preview }, onProgressHandler, onProgressErrorHandler, false, 5000); }; + debug('livePreview start:', dateStart); start(id_task, 0); } diff --git a/modules/progress.py b/modules/progress.py index d18d1ee9f..68102b915 100644 --- a/modules/progress.py +++ b/modules/progress.py @@ -1,4 +1,5 @@ import base64 +import os import io import time from pydantic import BaseModel, Field # pylint: disable=no-name-in-module @@ -10,6 +11,8 @@ finished_tasks = [] recorded_results = [] recorded_results_limit = 2 +debug = os.environ.get('SD_PREVIEW_DEBUG', None) is not None +debug_log = shared.log.trace if debug else lambda *args, **kwargs: None def start_task(id_task): @@ -48,6 +51,7 @@ class InternalProgressResponse(BaseModel): queued: bool = Field(title="Whether the task is in queue") paused: bool = Field(title="Whether the task is paused") completed: bool = Field(title="Whether the task has already finished") + debug: bool = Field(title="Debug logging level") progress: float = Field(default=None, title="Progress", description="The progress with a range of 0 to 1") eta: float = Field(default=None, title="ETA in secs") live_preview: str = Field(default=None, title="Live preview image", description="Current live preview; a data: uri") @@ -60,8 +64,6 @@ def progressapi(req: ProgressRequest): queued = req.id_task in pending_tasks completed = req.id_task in finished_tasks paused = shared.state.paused - if not active: - return InternalProgressResponse(job=shared.state.job, active=active, queued=queued, paused=paused, completed=completed, id_live_preview=-1, textinfo="Queued..." if queued else "Waiting...") shared.state.job_count = max(shared.state.frame_count, shared.state.job_count, shared.state.job_no) batch_x = max(shared.state.job_no, 0) batch_y = max(shared.state.job_count, 1) @@ -75,14 +77,17 @@ def progressapi(req: ProgressRequest): eta = predicted - elapsed if predicted is not None else None id_live_preview = req.id_live_preview live_preview = None - shared.state.set_current_image() - if shared.opts.live_previews_enable and (shared.state.id_live_preview != req.id_live_preview) and (shared.state.current_image is not None): + updated = shared.state.set_current_image() + debug_log(f'Preview: job={shared.state.job} active={active} progress={current}/{total} request={id_live_preview} last={shared.state.id_live_preview} enabled={shared.opts.live_previews_enable} updated={updated} image={shared.state.current_image} elapsed={elapsed:.3f}') + if not active: + return InternalProgressResponse(job=shared.state.job, active=active, queued=queued, paused=paused, completed=completed, id_live_preview=-1, debug=debug, textinfo="Queued..." if queued else "Waiting...") + if shared.opts.live_previews_enable and (shared.state.id_live_preview != id_live_preview) and (shared.state.current_image is not None): buffered = io.BytesIO() shared.state.current_image.save(buffered, format='jpeg') live_preview = f'data:image/jpeg;base64,{base64.b64encode(buffered.getvalue()).decode("ascii")}' - id_live_preview = shared.state.id_live_preview + id_live_preview = shared.state.id_live_preview - res = InternalProgressResponse(job=shared.state.job, active=active, queued=queued, paused=paused, completed=completed, progress=progress, eta=eta, live_preview=live_preview, id_live_preview=id_live_preview, textinfo=shared.state.textinfo) + res = InternalProgressResponse(job=shared.state.job, active=active, queued=queued, paused=paused, completed=completed, progress=progress, eta=eta, live_preview=live_preview, id_live_preview=id_live_preview, debug=debug, textinfo=shared.state.textinfo) return res diff --git a/modules/shared_state.py b/modules/shared_state.py index 024427f09..3fe0279f1 100644 --- a/modules/shared_state.py +++ b/modules/shared_state.py @@ -149,16 +149,17 @@ def end(self, api=None): def set_current_image(self): if self.job == 'VAE': # avoid generating preview while vae is running - return + return False from modules.shared import opts, cmd_opts - if cmd_opts.lowvram or self.api or not opts.live_previews_enable or opts.show_progress_every_n_steps <= 0: - return - if not self.disable_preview and (abs(self.sampling_step - self.current_image_sampling_step) >= opts.show_progress_every_n_steps): - self.do_set_current_image() + if cmd_opts.lowvram or self.api or (not opts.live_previews_enable) or (opts.show_progress_every_n_steps <= 0): + return False + if (not self.disable_preview) and (abs(self.sampling_step - self.current_image_sampling_step) >= opts.show_progress_every_n_steps): + return self.do_set_current_image() + return False def do_set_current_image(self): - if self.current_latent is None or self.disable_preview or self.preview_busy: - return + if (self.current_latent is None) or self.disable_preview or self.preview_busy: + return False from modules import shared, sd_samplers self.preview_busy = True try: @@ -175,10 +176,13 @@ def do_set_current_image(self): """ image = sd_samplers.samples_to_image_grid(sample) if shared.opts.show_progress_grid else sd_samplers.sample_to_image(sample) self.assign_current_image(image) + self.preview_busy = False + return True except Exception as e: + self.preview_busy = False log.error(f'State image: last={self.id_live_preview} step={self.sampling_step} {e}') display(e, 'State image') - self.preview_busy = False + return False def assign_current_image(self, image): self.current_image = image From dcdd6f20845c9336fac887486c1575714f4fc806 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Mon, 30 Dec 2024 16:44:07 +0300 Subject: [PATCH 175/249] Fix quanto logging with model offload --- modules/model_quant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/model_quant.py b/modules/model_quant.py index 10e5eb99e..d570c6ced 100644 --- a/modules/model_quant.py +++ b/modules/model_quant.py @@ -102,7 +102,7 @@ def load_quanto(msg='', silent=False): quanto = optimum_quanto fn = f'{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access log.debug(f'Quantization: type=quanto version={quanto.__version__} fn={fn}') # pylint: disable=protected-access - if shared.opts.diffusers_offload_mode != 'none': + if shared.opts.diffusers_offload_mode in {'balanced', 'sequential'}: shared.log.error(f'Quantization: type=quanto offload={shared.opts.diffusers_offload_mode} not supported') return quanto except Exception as e: From 28f966d7864007a771df5e76401737489a407a50 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Mon, 30 Dec 2024 16:49:58 +0300 Subject: [PATCH 176/249] Fix live previews with Flux --- modules/processing_callbacks.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/modules/processing_callbacks.py b/modules/processing_callbacks.py index 048f53d52..a9c6fbcf7 100644 --- a/modules/processing_callbacks.py +++ b/modules/processing_callbacks.py @@ -95,6 +95,10 @@ def diffusers_callback(pipe, step: int = 0, timestep: int = 0, kwargs: dict = {} if kwargs[key] is not None: kwargs[key] = kwargs[key].chunk(2)[-1] try: + current_noise_pred = kwargs.get("noise_pred", None) + if current_noise_pred is None: + current_noise_pred = kwargs.get("predicted_image_embedding", None) + if hasattr(pipe, "_unpack_latents") and hasattr(pipe, "vae_scale_factor"): # FLUX if p.hr_resize_mode > 0 and (p.hr_upscaler != 'None' or p.hr_resize_mode == 5) and p.is_hr_pass: width = max(getattr(p, 'width', 0), getattr(p, 'hr_upscale_to_x', 0)) @@ -103,12 +107,14 @@ def diffusers_callback(pipe, step: int = 0, timestep: int = 0, kwargs: dict = {} width = getattr(p, 'width', 0) height = getattr(p, 'height', 0) shared.state.current_latent = pipe._unpack_latents(kwargs['latents'], height, width, pipe.vae_scale_factor) # pylint: disable=protected-access + if current_noise_pred is not None: + shared.state.current_noise_pred = pipe._unpack_latents(current_noise_pred, height, width, pipe.vae_scale_factor) # pylint: disable=protected-access + else: + shared.state.current_noise_pred = current_noise_pred else: shared.state.current_latent = kwargs['latents'] + shared.state.current_noise_pred = current_noise_pred - shared.state.current_noise_pred = kwargs.get("noise_pred", None) - if shared.state.current_noise_pred is None: - shared.state.current_noise_pred = kwargs.get("predicted_image_embedding", None) if hasattr(pipe, "scheduler") and hasattr(pipe.scheduler, "sigmas") and hasattr(pipe.scheduler, "step_index"): shared.state.current_sigma = pipe.scheduler.sigmas[pipe.scheduler.step_index - 1] shared.state.current_sigma_next = pipe.scheduler.sigmas[pipe.scheduler.step_index] From 77cc16786c5331122334ca8926b5d7576382a5bf Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 30 Dec 2024 09:06:32 -0500 Subject: [PATCH 177/249] fix hires batch, sdxl refiner Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 2 + configs/sdxl-refiner/model_index.json | 35 + .../scheduler/scheduler_config.json | 18 + .../sdxl-refiner/text_encoder_2/config.json | 24 + configs/sdxl-refiner/tokenizer_2/merges.txt | 48895 +++++++++++++++ .../tokenizer_2/special_tokens_map.json | 24 + .../tokenizer_2/tokenizer_config.json | 33 + configs/sdxl-refiner/tokenizer_2/vocab.json | 49410 ++++++++++++++++ configs/sdxl-refiner/unet/config.json | 69 + configs/sdxl-refiner/vae/config.json | 32 + modules/processing_args.py | 11 +- modules/processing_diffusers.py | 7 +- modules/sd_detect.py | 2 + modules/shared_items.py | 2 +- .../textual_inversion/textual_inversion.py | 2 +- 15 files changed, 98555 insertions(+), 11 deletions(-) create mode 100644 configs/sdxl-refiner/model_index.json create mode 100644 configs/sdxl-refiner/scheduler/scheduler_config.json create mode 100644 configs/sdxl-refiner/text_encoder_2/config.json create mode 100644 configs/sdxl-refiner/tokenizer_2/merges.txt create mode 100644 configs/sdxl-refiner/tokenizer_2/special_tokens_map.json create mode 100644 configs/sdxl-refiner/tokenizer_2/tokenizer_config.json create mode 100644 configs/sdxl-refiner/tokenizer_2/vocab.json create mode 100644 configs/sdxl-refiner/unet/config.json create mode 100644 configs/sdxl-refiner/vae/config.json diff --git a/CHANGELOG.md b/CHANGELOG.md index df9a98edd..621f65daf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,8 @@ - interrogate caption with T5 - on-the-fly quantization using TorchAO - remove concurrent preview requests + - hires batch + - sdxl refiner ## Update for 2024-12-24 diff --git a/configs/sdxl-refiner/model_index.json b/configs/sdxl-refiner/model_index.json new file mode 100644 index 000000000..d1024c840 --- /dev/null +++ b/configs/sdxl-refiner/model_index.json @@ -0,0 +1,35 @@ +{ + "_class_name": "StableDiffusionXLImg2ImgPipeline", + "_diffusers_version": "0.19.0.dev0", + "force_zeros_for_empty_prompt": false, + "add_watermarker": null, + "requires_aesthetics_score": true, + "scheduler": [ + "diffusers", + "EulerDiscreteScheduler" + ], + "text_encoder": [ + null, + null + ], + "text_encoder_2": [ + "transformers", + "CLIPTextModelWithProjection" + ], + "tokenizer": [ + null, + null + ], + "tokenizer_2": [ + "transformers", + "CLIPTokenizer" + ], + "unet": [ + "diffusers", + "UNet2DConditionModel" + ], + "vae": [ + "diffusers", + "AutoencoderKL" + ] +} diff --git a/configs/sdxl-refiner/scheduler/scheduler_config.json b/configs/sdxl-refiner/scheduler/scheduler_config.json new file mode 100644 index 000000000..e5bc8421e --- /dev/null +++ b/configs/sdxl-refiner/scheduler/scheduler_config.json @@ -0,0 +1,18 @@ +{ + "_class_name": "EulerDiscreteScheduler", + "_diffusers_version": "0.19.0.dev0", + "beta_end": 0.012, + "beta_schedule": "scaled_linear", + "beta_start": 0.00085, + "clip_sample": false, + "interpolation_type": "linear", + "num_train_timesteps": 1000, + "prediction_type": "epsilon", + "sample_max_value": 1.0, + "set_alpha_to_one": false, + "skip_prk_steps": true, + "steps_offset": 1, + "timestep_spacing": "leading", + "trained_betas": null, + "use_karras_sigmas": false +} diff --git a/configs/sdxl-refiner/text_encoder_2/config.json b/configs/sdxl-refiner/text_encoder_2/config.json new file mode 100644 index 000000000..da1848b5e --- /dev/null +++ b/configs/sdxl-refiner/text_encoder_2/config.json @@ -0,0 +1,24 @@ +{ + "architectures": [ + "CLIPTextModelWithProjection" + ], + "attention_dropout": 0.0, + "bos_token_id": 0, + "dropout": 0.0, + "eos_token_id": 2, + "hidden_act": "gelu", + "hidden_size": 1280, + "initializer_factor": 1.0, + "initializer_range": 0.02, + "intermediate_size": 5120, + "layer_norm_eps": 1e-05, + "max_position_embeddings": 77, + "model_type": "clip_text_model", + "num_attention_heads": 20, + "num_hidden_layers": 32, + "pad_token_id": 1, + "projection_dim": 1280, + "torch_dtype": "float16", + "transformers_version": "4.32.0.dev0", + "vocab_size": 49408 +} diff --git a/configs/sdxl-refiner/tokenizer_2/merges.txt b/configs/sdxl-refiner/tokenizer_2/merges.txt new file mode 100644 index 000000000..76e821f1b --- /dev/null +++ b/configs/sdxl-refiner/tokenizer_2/merges.txt @@ -0,0 +1,48895 @@ +#version: 0.2 +i n +t h +a n +r e +a r +e r +th e +in g +o u +o n +s t +o r +e n +o n +a l +a t +e r +i t +i n +t o +r o +i s +l e +i c +a t +an d +e d +o f +c h +o r +e s +i l +e l +s t +a c +o m +a m +l o +a n +a y +s h +r i +l i +t i +f or +n e +ð Ł +r a +h a +d e +o l +v e +s i +u r +a l +s e +' s +u n +d i +b e +l a +w h +o o +d ay +e n +m a +n o +l e +t o +ou r +i r +g h +w it +i t +y o +a s +s p +th is +t s +at i +yo u +wit h +a d +i s +a b +l y +w e +th e +t e +a s +a g +v i +p p +s u +h o +m y +. . +b u +c om +s e +er s +m e +m e +al l +c on +m o +k e +g e +ou t +en t +c o +f e +v er +a r +f ro +a u +p o +c e +gh t +ar e +s s +fro m +c h +t r +ou n +on e +b y +d o +t h +w or +er e +k e +p ro +f or +d s +b o +t a +w e +g o +h e +t er +in g +d e +b e +ati on +m or +a y +e x +il l +p e +k s +s c +l u +f u +q u +v er +ðŁ ĺ +j u +m u +at e +an d +v e +k ing +m ar +o p +h i +.. . +p re +a d +r u +th at +j o +o f +c e +ne w +a m +a p +g re +s s +d u +no w +y e +t ing +y our +it y +n i +c i +p ar +g u +f i +a f +p er +t er +u p +s o +g i +on s +g r +g e +b r +p l +' t +m i +in e +we e +b i +u s +sh o +ha ve +to day +a v +m an +en t +ac k +ur e +ou r +â Ģ +c u +l d +lo o +i m +ic e +s om +f in +re d +re n +oo d +w as +ti on +p i +i r +th er +t y +p h +ar d +e c +! ! +m on +mor e +w ill +t ra +c an +c ol +p u +t e +w n +m b +s o +it i +ju st +n ing +h ere +t u +p a +p r +bu t +wh at +al ly +f ir +m in +c a +an t +s a +t ed +e v +m ent +f a +ge t +am e +ab out +g ra +no t +ha pp +ay s +m an +h is +ti me +li ke +g h +ha s +th an +lo ve +ar t +st e +d ing +h e +c re +w s +w at +d er +it e +s er +ac e +ag e +en d +st r +a w +st or +r e +c ar +el l +al l +p s +f ri +p ho +p or +d o +a k +w i +f re +wh o +sh i +b oo +s on +el l +wh en +il l +ho w +gre at +w in +e l +b l +s si +al i +som e +ðŁ Ĵ +t on +d er +le s +p la +ï ¸ +e d +s ch +h u +on g +d on +k i +s h +an n +c or +. . +oun d +a z +in e +ar y +fu l +st u +ou ld +st i +g o +se e +ab le +ar s +l l +m is +b er +c k +w a +en ts +n o +si g +f e +fir st +e t +sp e +ac k +i f +ou s +' m +st er +a pp +an g +an ce +an s +g ood +b re +e ver +the y +t ic +com e +of f +b ack +as e +ing s +ol d +i ght +f o +h er +happ y +p ic +it s +v ing +u s +m at +h om +d y +e m +s k +y ing +the ir +le d +r y +u l +h ar +c k +t on +on al +h el +r ic +b ir +vi e +w ay +t ri +d a +p le +b ro +st o +oo l +ni ght +tr u +b a +re ad +re s +ye ar +f r +t or +al s +c oun +c la +t ure +v el +at ed +le c +en d +th ing +v o +ic i +be st +c an +wor k +la st +af ter +en ce +p ri +p e +e s +i l +âĢ ¦ +d re +y s +o ver +i es +ðŁ ij +com m +t w +in k +s un +c l +li fe +t t +a ch +l and +s y +t re +t al +p ol +s m +du c +s al +f t +' re +ch e +w ar +t ur +ati ons +ac h +m s +il e +p m +ou gh +at e +st ar +wee k +! !! +c lu +th ere +n er +t om +s el +ï¸ ı +wor ld +v es +c am +go t +in ter +of f +u m +ton ight +o ther +h ou +loo k +j e +i d +si on +be au +at t +el i +or t +re c +f f +st er +su pp +g en +be en +il y +te am +m m +i c +pe op +it t +at s +on ly +mb er +en g +b ri +m p +k now +b ur +b ar +in s +lo w +sh e +ro w +â Ŀ +t ro +peop le +vi a +lo w +ag a +be t +x t +f ac +ch ar +e ar +w al +s en +f am +b le +n ati +is h +n or +g ame +li ve +s co +le y +d on +ic k +b all +ver y +the se +p an +i a +at ing +c r +a re +g ir +ma ke +st re +sho w +. " +f l +u p +d r +than ks +il li +w om +st s +i g +s ur +ever y +c ur +vie w +le t +in to +mo st +n a +in di +g ar +ha d +s ou +v ed +an t +iti on +ma de +f ol +un i +it ed +ðŁ ı +ic al +th r +read y +ch ec +d ra +k es +boo k +e p +si c +mor ning +ne ws +c au +c t +w ell +an c +pho to +th an +or s +bir th +g g +ou t +ne xt +som e +en ing +stor y +ch ri +do wn +hom e +f fe +fre e +d a +b or +f il +ci al +than k +si de +le ar +qu e +l ine +t en +at es +ye ars +m y +pho to +beau ti +ri ght +n u +for m +shi p +b an +th er +d ays +g am +as on +g y +ðŁ İ +birth day +se t +ic k +e t +st ill +com ing +ta ke +ðŁ ĩ +b b +s ol +s on +d en +e p +mu sic +the m +de n +wh y +f oo +c ra +am az +w n +h ol +t ting +w r +u e +ma g +c ro +l an +c lo +b ra +a k +s ing +c al +re ad +' ve +jo h +b ab +d ri +b lo +bi g +er ic +in t +t or +tr y +l a +le g +hou se +m ic +v al +beauti ful +l itt +chec k +ne w +ver s +s w +ar i +pla y +h er +âĢ ĵ +w in +m a +con gr +sch ool +f un +. @ +he al +ic h +d el +wh ere +l on +ke t +tw o +mu ch +wat ch +v en +d ed +a st +k ed +b as +go ing +m p +e ver +w ays +ro o +de sig +l y +s ed +to p +l in +ch an +to o +it ing +d ent +gh ts +t y +sp o +ne ed +b lu +in st +be ing +âĿ ¤ +w el +l s +hi m +m ay +st ing +n a +el y +litt le +g a +n at +tom or +m c +h on +w ant +a ir +pi c +am eric +p er +le ss +wee k +ve l +a h +c ap +ch am +g er +ti m +tomor row +ne ss +st ate +h al +ser v +z e +o s +p at +v is +ex c +s in +f f +c ity +c en +an y +b el +su mm +t in +w ould +loo king +k o +ce le +fam ily +m er +po w +hel p +bu s +c o +c le +sel f +en s +ic s +th o +an i +ch o +le ad +b s +t wee +th ink +for e +ch il +vi de +di d +al e +ch i +v il +en ds +w ing +p as +' ll +v ol +s a +g s +man y +j ec +be fore +gra ph +n y +ur ing +w il +d d +bu il +f av +st ed +tr an +l ing +ou d +d ge +fi el +nati onal +st a +c er +w ere +in a +se ason +c ou +n ed +amaz ing +ti ons +cele br +n s +a th +he ad +s day +d ar +lo c +v in +an other +g oo +s at +n y +jo in +pre s +s es +s ing +an a +in ing +.. .. +c our +ï¸ ı +ac t +cau se +li ght +am s +t a +b al +f c +hi gh +off ici +t t +chri st +d ic +d ay +ra l +h or +: ) +vi si +n am +o b +ma s +gh t +re ally +t un +fin d +thr ough +por t +u t +ti ve +st y +n e +or e +ðŁĺ Ĥ +supp ort +ne ver +ev en +ðŁ Ķ +h a +y a +l d +u k +r an +j am +wi th +me di +d es +ne y +ch ing +al e +h y +k in +! ! +d y +pl ace +al so +b le +wh ich +bl ack +b li +s ay +par k +pl ay +ir e +vide o +week end +a il +ke y +p t +w ard +fri day +d in +ine ss +g ro +b en +al ways +t ball +ag o +m il +c y +pro duc +di sc +un der +ple ase +sp or +fu ll +e y +ðŁ Ļ +is e +iti es +c at +k no +u se +fo re +k er +ar t +hi gh +op en +s an +e f +our s +sh ed +st ri +d ro +aga in +i m +ðŁ ĵ +en jo +fu n +ge tting +p en +g er +c li +an y +ever y +e u +wom en +â ľ +e st +c ould +r y +" @ +th ou +sh a +comm un +b er +d ents +di s +wh ile +aw ay +di o +h am +g la +d ate +k a +mis s +un ch +w on +in f +roo m +g a +re al +ex per +di rec +sh ould +sp r +g ol +l ong +bet ter +or i +e y +i ence +il s +z z +h an +f ound +v s +â Ļ +po st +ti c +par t +m en +ren ce +ce ss +v ic +s il +sho p +ðŁĺ Ĥ +f ood +v al +sti c +y ou +s ays +e lec +st ar +o c +l and +i d +c tion +fiel d +s of +st art +wat er +fri ends +on es +ðŁ Į +f la +f ar +wh ite +par ty +in st +gr ou +t v +every one +m ent +j a +ch a +pr in +an ts +d uring +l at +l ar +we st +th en +k a +y oun +in sp +in te +we en +visi t +aga inst +re le +he ad +c es +to wn +loo ks +th re +re gi +ren t +pro jec +gir l +se ar +w o +m om +c ar +h un +pu bli +d i +p le +c all +c ri +u m +for d +per fe +fri end +h ard +ssi on +te st +pla ying +ar ound +be cause +ke ts +me et +sat ur +ar ti +wor k +j un +v en +r un +me mber +por t +su per +t wit +s am +el s +t ly +ad v +ati ve +at h +s ure +av ail +la r +s qu +ar ds +ev ent +m en +l l +o ver +lo gy +it al +tim es +m al +b ack +c oo +ma king +st ru +â ģ +it u +sh ar +g an +c as +s n +summ er +pic ture +f an +h in +christ mas +c y +pr oud +cham pi +desig n +pp ing +ho pe +c a +avail able +ma y +we d +photo graph +spe cial +sal e +sto p +er y +a we +al ity +hi story +am a +pre si +b ru +wor king +d one +d r +k en +fe at +w ood +ate st +sun day +mo vi +vel y +s le +f ace +sp ec +stu dents +b y +ha m +sp on +bus iness +d at +i e +i p +so ci +g lo +h and +re cor +r s +me e +ke ep +p ur +heal th +sh e +com ple +go d +da vi +col lec +li st +r a +clu b +t ers +in clu +th ings +pl an +â ĺ +joh n +sh ing +at ul +so on +blu e +g or +satur day +w on +congr atul +se e +âĿ¤ ï¸ı +tho se +ðŁĺ į +fin al +d ou +it h +o wn +ro ad +t our +a st +indi a +ti l +n d +f er +fav or +su l +lear n +fir e +ju st +grou p +a h +r ac +bo dy +u r +c are +à ¸ +p lo +o h +po s +gi ve +te ch +su b +c ent +er ing +y m +il ity +f ic +lon don +v ir +gu ys +b a +ðŁ ¤ +bab y +sc re +ðŁĺ į +tru mp +un der +chan ge +i an +col le +ss es +l er +ss ed +n ice +ann oun +pow er +s ar +a king +min i +s li +s wee +k ar +fu l +c ru +ac tion +a ther +) . +st and +de vel +a a +g an +le ft +lo l +re l +tran s +m ents +in t +e f +man ag +di g +gen er +do wn +p au +ti v +k u +th ur +k en +st on +f ans +tal k +twee t +t oo +sty le +pro te +se con +fr on +awe some +g l +p al +ne t +s or +la u +g on +sin ce +t ty +ser ies +me mor +b eli +fil m +di d +di es +o t +congratul ations +p ra +e ve +w oo +offici al +su c +in cre +b on +par t +pp ed +cla ss +si ve +bo y +cu l +perfe ct +t ou +d am +wel come +foo tball +h i +p ap +wa it +ad a +congr ats +youn g +exc ited +re ce +j an +v a +re d +st ra +medi a +' d +do es +le t +mu l +ill s +gre en +m el +to ge +fu ture +ye ster +vers ity +for m +ta in +i de +ch es +ki ds +qu i +ha ha +de ta +bi g +favor ite +gir ls +con tin +do m +sear ch +u al +a ir +d ers +mon th +c er +yester day +commun ity +ad e +do g +vil le +ic es +d eli +sy ste +ru n +is m +he art +c up +en ti +fe w +presi dent +e ds +un til +fe sti +o k +f lo +sa id +ol e +me d +tra vel + £ +ph one +toge ther +fa st +lo t +gam es +sh ir +bet ween +y es +th ers +do ing +m ac +at or +b and +fol low +projec t +devel op +di ffe +con fe +spe ci +ca st +y s +bo ard +r d +i al +sh oo +r am +ha ving +sh are +fol low +on e +n ame +m r +pu t +disc u +or y +c ame +ou s +s ite +twit ter +t b +t it +fin ally +z ed +su per +com pan +us ing +all s +li st +r is +sho t +g al +t ar +de l +joh n +âĢ Ķ +some thing +ra m +inte re +wh e +b it +ðŁ į +stre et +oun d +a i +tic kets +movi e +re al +k y +ta king +o pp +c c +l am +m oun +in ve +bl ack +us ed +on line +y or +loc al +gu e +c ks +o w +ge st +bo ys +illi on +con t +re ci +in ed +eu ro +no w +se en +p h +te ach +de f +sou th +su ch +aw ard +mu st +is su +ca re +fe el +p lu +l atest +spor ts +we b +te x +e ment +s k +fi c +w an +te ch +o t +bo x +n er +fre e +t al +a sh +c ase +ho t +won der +mee ting +er a +ch all +ðŁ IJ +jo b +il i +c ool +j our +th s +m o +f el +di e +mic ha +e le +te am +serv ice +st and +ma kes +p ing +ear ly +com es +e k +ho li +v ers +ag ue +s au +thre e +mon day +fa shi +some one +th ro +se a +b ad +supp or +tur n +ur y +m ing +photograph y +n ic +mar k +pre tty +ss ing +wat ching +me mb +ar ri +coun ty +be ach +fr an +cen ter +pol ice +b at +publi c +t an +pre ss +s af +s y +ge ts +ro y +n ers +y our +bu y +st ers +sho w +as ed +chil dre +af ric +in es +sp ace +sc ri +h all +pa in +ar ing +hom e +m ur +heal th +ch ed +s and +rece i +gu y +e a +americ an +re si +childre n +- - +i ri +ing ton +coun try +ro ss +le n +ann a +boo ks +b c +e ce +d om +lo vely +k h +pe t +g y +g ri +st age +off ice +ro ck +m on +b ay +t able +su n +m ed +th in +l or +f low +( @ +uni versity +stor e +fron t +goo d +z a +vo te +nor th +he y +an im +or der +mi d +with out +a de +re member +mar ket +? ? +mu s +tra ining +e duc +bu t +co ver +st an +sc en +b la +bre ak +l ou +s ame +g old +a in +o s +bo th +l it +ver n +a i +al bu +p a +enjo y +be g +ell ing +thur sday +inf o +s an +americ a +ha ir +te l +mar ch +con cer +colle ge +confe rence +ap p +h our +ch ang +â ļ +s our +ol s +we ather +w ar +p hi +festi val +secon d +cu te +pr ac +en er +str y +le a +pol it +s av +se n +o w +m i +ne ar +ou ght +z e +co ffe +w illi +d an +se y +davi d +e se +f an +de ci +the at +no v +ati on +tr ac +sc i +re view +c el +e m +u n +ju ly +or ig +ti on +d ru +form er +st ay +af ter +in v +too k +dat a +b al +tu es +d an +ev ening +ðŁĺĤ ðŁĺĤ +d ol +u res +pro vi +t s +e st +sig n +j ac +u k +s ong +ye t +bo w +in du +j ap +h oo +po int +any one +z y +i st +h ur +it al +buil ding +wom an +ch ur +j er +per for +co ach +le ague +ce ss +ne t +i mag +nati on +br it +qu e +aw ards +ag es +wor ks +c ed +man ce +l ate +ig n +mon ey +tru e +i i +t ell +pl ac +p ac +as y +wor ld +be hin +im port +read ing +gra m +gi ving +me t +h it +for ward +st om +pres ent +jun e +so cial +no on +mar t +hal f +s we +go vern +k er +deta ils +li sh +_ _ +ac y +si a +ber t +f all +! !!! +) , +th i +d iti +sp ort +k ing +f it +st af +c at +mu se +cen tr +y er +con tro +b loo +wal k +ac tu +did n +li m +lear ning +re search +wed ne +au th +h ours +k y +f ar +h en +.. .. +it ch +ri l +str ong +sk y +que sti +jam es +r on +d g +f ur +c in +do es +app ro +mar ke +tu res +ful ly +ch at +behin d +te m +fin i +mis sion +b att +fe el +he av +every thing +b ar +w ish +pre mi +i ma +exper ience +e ach +re port +swee t +tic s +spr ing +re spon +syste m +vic tor +l in +sa w +al ready +gh ter +f le +ã ĥ +br ing +albu m +- - +ell s +st an +to m +inter national +w ent +an ni +mat ch +pp er +st one +sm all +ra in +fashi on +are a +v an +ag ram +k o +thou ght +wor th +v an +m er +coffe e +it es +g n +arti st +c on +ar ch +c ir +se cre +gr ound +is o +h and +co m +bri dge +h s +x i +l ink +pu l +sp l +r ace +f li +ri ver +g as +di sco +d al +play er +f it +photo s +it y +o k +j or +tr a +ap ril +ad s +a di +sol u +beau ty +do or +me ss +up date +ali a +sch o +en ed +mom ent +sco t +sc ience +i or +ti es +ac ross +ous ly +sh es +does n +p age +wat er +m illion +cla ssi +l ic +ca st +form ation +micha el +ell o +s mo +in ts +vi sion +op ening +ld n +au str +tues day +win ner +po ssi +r ound +shir t +di t +b o +u es +il led +al ong +tri p +star ting +im pro +k an +per son +no t +re co +ne eds +c le +li e +re st +r ing +win ter +si mp +mo m +be er +fac e +tor s +us a +collec tion +ge or +se ssion +tr ying +la s +la ke +j en +orig in +stu dent +se cur +v in +pic s +ex pe +com p +gon na +e qu +b ad +le y +a u +memb ers +bre ak +w all +gi c +din ner +bu l +insp ir +r i +min d +ic a +win ning +tal king +t ren +s is +t en +wonder ful +s now +he ar +th om +no thing +gu i +st in +blo g +fe st +b un +le e +war ds +ch ance +dre ss +re n +pau l +p es +tech no +ru ssi +c ard +e ast +mar i +w ine +t i +la w +str ic +k i +ap e +au gu +pro fe +as h +cour se +ma il +ren tly +d un +m un +lo ve +is land +dri ve +s l +end ed +ma in +lo st +nat ure +âĿ¤ ï¸ı +ch ic +re por +p in +pr o +st ation +ce p +ta kes +compan y +go es +on d +ma ch +ra dio +d ad +ro ck +j a +p ay +champi on +e e +in de +tt a +ati c +t ab +beli eve +ener gy +z i +t at +wor d +on ce +re sul +y l +and re +an o +inst agram +clo se +t am +cu stom +w a +con om +sho ws +li fe +k in +ro b +t age +n ation +al most +list en +sa ve +re li +ac e +mar y +tre e +for get +j ack +wa iting +direc tor +h ill +bor n +te mp +f l +st e +on a +sing le +wedne sday +un ited +in o +@ _ +ne l +celebr ate +en ding +de al +j i +can ada +hu ge +tr ack +âĢ ¢ +f y +fan ta +an g +yor k +rele ase +p un +ep iso +wor ds +t our +p ack +i gh +classi c +perfor mance +ke t +after noon +recor d +win s +pro ble +âĿ ¤ +f our +b ed +ban k +d ance +s la +cal led +mi ght +a p +pa st +ðŁ ļ +diffe rent +it e +gi ft +ssi ve +chur ch +c us +pro gram +ho tel +ic e +ma d +secur ity +en ge +d c +en ough +st a +e ty +de ad +g un +he ar +m ir +hu man +gre ss +oun ds +pi ece +bre aking +gar den +fi ght +vie ws +f ish +star ted +run ning +gre en +ser i +s m +as k +d or +de ath +e conom +er i +ir d +s er +l unch +âģ ¦ +bo x +nat u +ba se +b an +f al +glo bal +wil d +wo w +out side +mo ve +le ad +an al +muse um +on g +ha w +pow er +than k +b ac +char ac +cam pa +dig ital +r o +op er +de v +w ol +p ati +f a +m ale +pap er +ill ing +c s +â ĥ +educ ation +ta ken +e ffe +m ou +s ad +" . +bas ed +staf f +inclu ding +li ving +a c +ch ina +mo b +stor m +lu ck +ph il +o o +y n +tra vel +k el +ti al +pr ice +boo k +import ant +bi o +p ool +ny c +f ab +lo ad +? ! +chall enge +cr y +ser ve +we ar +bu s +ta in +nu mber +ro r +k at +i z +th ough +ho sp +m m +fa ir +ut es +ho t +po p +fi ed +cam p +develop ment +li br +c ali +em s +âģ¦ @ +b ol +is ed +stand ing +mo del +it a +g le +bro wn +ima ge +ve red +for ce +o il +par tic +sh u +da ily +la w +se c +cla ss +cam p +holi day +cl in +k ers +pres ent +gam e +incre di +er ship +inter view +b ill +du e +and y +ab o +in nov +ke y +ac ade +p il +mo der +st ars +br and +f er +wee ks +con si +pr e +sa fe +wr it +di um +la unch +marke ting +ann ual +as si +cour t +la dy +c ted +and a +in side +chil d +opp or +sm ith +centr e +gu e +âģ © +f ren +st y +for t +ent ly +is n +ke ep +to ber +on y +bo y +al d +col la +de mo +le vel +com pet +ad o +b our +fanta stic +m ate +s u +sou th +oppor tun +vers ary +lat er +bu d +face book +la un +ster n +p it +! " +ma j +gr am +tb t +fi re +happ y +a ks +wh ole +actu ally +ill er +ell a +lo ts +al ex +an ge +lan ds +ðŁĺ Ń +en ter +r ou +episo de +p ed +in ten +sh ire +wh o +pl an +h o +ca ke +we st +mag az +fre sh +c c +n ar +ch ris +wr iting +w er +n om +l o +mi dd +dre am +o l +ti onal +de b +> > +be come +s i +gr and +all ing +hi stor +ri de +i red +saf e +que en +ci l +in tro +vi l +d ani +.. . +ar tic +st at +sh ort +or ing +sel fi +mis si +do c +b it +g all +b om +i re +se lec +d ition +ðŁĶ ¥ +fri end +be at +gh ting +ðŁĺ Ĭ +pe ace +ex hi +ant a +ab ility +il lu +j on +qu ality +tri bu +m es +play ers +fa ir +cu t +c ab +suc cess +b i +su s +pro mo +sch e +an ge +ic o +comm it +cat ch +ill a +kin d +feel ing +qu o +s ay +anni versary +spo t +mo ther +an e +p end +your self +op s +app le +min utes +p o +gr and +ri es +ha ha +care er +ed ition +de c +ric k +am i +concer t +iti ve +ge ous +d ly +t te +adv ent +i g +li ghts +ak er +sk y +âĥ £ +r ay +fini shed +w ay +s d +ac coun +ðŁĴ ķ +ck y +ch el +lit er +pain ting +lo s +st un +techno logy +n as +ma r +b il +afric a +ki e +ey es +gol f +plu s +ni a +it ec +serv ices +wed ding +kno wn +te le +.. ... +star ts +pa ren +w ants +ati onal +mon ths +win do +fav our +er t +magaz ine +ex clu +re ve +b c +origin al +e ss +n al +an ti +st ro +t ice +stu dy +à ¤ +v ac +nation al +fi ve +ra in +ve ment +u te +ver se +em er +ar my +possi ble +gue ss +val ley +ther n +cro w +m r +col or +on to +pic k +cle ar +dar k +t ac +wan ted +it ting +can cer +govern ment +di e +ri se +z ing +col d +f oun +stu dio +str ation +bro ther +a head +sh el +mic ro +ic ally +d au +sig ned +vi ol +a x +as se +i o +w re +spl ay +ch ick +augu st +pl at +ti ps +sp i +hu man +e asy +lo gi +mi ke +gro w +ag re +w w +sh ad +mo tiv +wi de +tur ns +om g +v ar +de fin +su g +j im +ðŁĶ ¥ +t d +campa ign +nam ed +re tweet +co p +t v +le av +k is +dou ble +s mar +issu e +vil la +in formation +li es +sto ck +n t +di stric +sh or +mi x +er o +se p +me x +see ing +li ve +re min +co de +g ur +s c +wil d +l un +h ood +spo t +fa ther +fore ver +up d +tra f +f ly +ne ed +gra du +tra in +ma ke +s ab +be y +si ze +lead er +tal ks +e u +lo g +fo x +gor geous +le ss +le ts +sur pri +my self +no te +li ves +f ru +lo ved +se ver +de m +j i +so c +h old +do gs +n i +â ŀ +lea ve +air port +ben ef +ex pl +shi ps +comple te +ach i +gre at +vin tage +j ack +ro c +woo d +pri v +off er +ey e +ver sion +te a +co ach +off ic +w ell +g en +s at +h h +you th +o x +? " +m t +mi x +g g +d le +natu ral +buil d +break fast +thin king +theat re +mo on +ber g +go als +geor ge +en e +exc ell +il ing +tun e +y ed +g ate +m it +net work +jo e +h ello +f b +tu be +we aring +ath le +stru c +har d +gla ss +g ers +thro w +g es +b t +indu stry +manag ement +ali st +go al +stre am +y el +a vi +ici ous +o thers +s ki +chri sti +bir d +e sc +m in +tr o +l t +j an +im p +ri ghts +sh a +or gan +cent ral +ar a +ro ll +favour ite +che ster +el se +p ay +car s +m ine +ste p +prac tice +maj or +h ang +ðŁĺ ĺ +n on +v ari +eng ine +vol un +di a +i led +arch itec +p ink +d s +th y +wa sh +web site +ba g +contro l +el li +f ra +an sw +d ence +y u +r on +ol a +g in +dr in +li c +cou ple +sp ar +g on +cre ate +c t +celebr ating +de ep +e at +te e +vo ice +dro p +vis it +at ors +sta dium +f t +w is +ro l +gra de +fam il +po ints +re pre +w as +traf fic +jap an +or g +hon or +tex as +man u +âĻ ¥ +safe ty +re r +b ag +em plo +rele ased +re gu +ak a +n av +ro le +sen ior +spec t +cro ss +lin es +be st +p ack +s in +ti e +mis sing +sun set +li ber +is ing +j ay +sk i +champion ship +ac tiv +la dies +play ed +y y +pu bl +al o +pri de +s r +pa ki +lu x +sur vi +ck ed +e ts +cho col +austr alia +par is +mi les +h at +ment al +al a +me an +mob ile +en a +in si +f ound +chi ef +t ag +incredi ble +re turn +à © +goo gle +fren ch +cre w +hal lo +ali an +j az +ch er +sil ver +nor th +eng lish +base ball +c af +lim ited +follow ing +app reci +ear th +k ir +ve mber +w ed +p tion +g ed +oc tober +fl ori +c r +en cy +ga ve +lor d +stu ff +ber ry +po st +sm ile +bro ad +st ate +gg er +me ans +ic y +gu n +y o +ma ster +bur g +han ds +ni e +/ / +uni on +brit ish +big gest +distric t +am ing +h il +o ce +per son +pas s +en vir +scho ols +arri ved +anc es +insp ired +ex pla +be n +libr ary +bo tt +am p +ste ph +cont act +b ang +m s +cali for +t old +batt le +b b +chic ago +âľ ¨ +str ate +sh i +de ce +- ) +ad d +la b +j ones +leg end +cast le +ing er +st ance +be l +ur a +re fu +lead ers +po t +se x +h ic +artic le +ki d +fr ance +x x +ex e +gui de +volun te +pr int +al i +ce o +twee ts +w x +scen e +vol u +ant i +h an +as soci +shar ing +ro se +mini ster +sh er +in ste +cle an +demo cr +po ster +sk in +p sy +pro per +cra zy +i am +o re +in i +any thing +po d +mo ving +cl ick +ex plo +com b +cra ft +f i +bloo d +is ra +publ ic +d ent +ol ym +eng land +a si +ch er +fac t +envir on +har ry +g one +me dic +enjo ying +just ice +j r +indi an +wi fe +s ound +t es +dra wing +p al +ide a +cr it +ju li +il er +war m +cl ar +thou ghts +def en +coun cil +intro duc +di ed +jan u +an i +s end +li er +m l +intere sting +tra de +win d +b ay +s ac +anc y +sour ce +b es +org ani +ar ly +lar ge +ff ici +ta g +u t +de sp +o es +tit le +sy m +pic tures +op en +wom en +sho wing +ri a +le ast +lead ership +cur rent +elec tr +val ent +list ening +c key +gener al +de ser +du ce +; ) +c ent +ðŁĺį ðŁĺį +sco tt +po or +selfi e +ev ents +i on +wr ong +de v +h ill +sep te +cul ture +l ine +sor ry +s ent +si ster +ce pt +k ri +no vember +ar i +announ ce +z ation +br an +g ent +d u +l en +per s +f m +mart in +o p +e mb +om e +midd le +suc cess +pe ter +janu ary +f lu +rac ing +d av +bi ke +ðŁı » +pe t +shoo t +profe ssi +feat uring +septe mber +now playing +sta ur +z a +on ic +qu ick +bas ke +spe aking +mil it +z er +chick en +b ell +s ad +co ast +lo ving +y ers +d j +pan el +ver age +s wit +ic ks +b ou +califor nia +s am +paren ts +er o +k illed +ph ys +jo bs +mi gr +an th +e mo +hallo ween +and er +c m +compet ition +e ag +s ket +sp ir +may be +exclu sive +app e +jour ney +scre en +for d +i o +h ate +u g +sou l +her o +soci ety +sy n +gu it +n h +d j +as es +im pre +ti me +sal es +d d +f ts +summ it +stun ning +om s +tur ned +cle an +sof t +be at +re staur +de red +en ces +ma gic +di o +sh ine +gu est +health y +exhi b +stor ies +po pu +n is +el a +bel ow +fun ny +resul ts +s ne +cur rently +ar d +down load +f light +m al +f ine +p ad +ch u +ent ed +h at +ðŁij ı +ste ve +j o +mar k +r at +b all +p c +p on +b by +o li +ar ts +as ure +bow l +att ack +mi c +de ar +ran ge +en ter +chocol ate +br illi +ac cess +, " +? ?? +ch ap +con st +t n +mat ter +blu e +gall ery +em p +work shop +lead ing +y ours +baske tball +w anna +th u +_ _ +mar ri +sle ep +bi a +ch e +ma d +imp act +o wn +si r +chan nel +euro pe +e sp +k itch +hosp ital +w ra +roy al +f s +ne u +qu ar +ne y +ac ks +ch ase +pp y +st al +at ely +ti m +dece mber +r are +per form +cre am +we ight +ch oo +ni ght +ha ven +fr anc +kh an +buil t +hel ping +tru st +ty pe +gol den +ta x +s now +s wi +di sa +questi ons +ve y +li ght +c n +cl oud +thom as +ag ed +sh ou +te ams +gr an +re ason +a a +you tube +v p +pi zz +manag er +bur y +cre dit +tre at +ma x +i k +ma in +g ing +de ad +pro bab +ye ah +ã Ĥ +br and +so li +pl ant +ta yl +gir l +ðŁĺ Ń +nam ent +au to +mess age +ko re +n ur +ter r +ag u +ma p +sen ting +lo ves +gi ves +g ab +z en +ro bert +con fir +w ars +o m +sta in +cam era +and er +won der +a b +ca p +s old +su it +wal king +contin ue +effe c +dau ghter +d anc +cha in +mul ti +ki d +y an +champi on +v o +ta ins +ho st +min i +mis sed +re sc +ly n +fin ish +del icious +s as +tayl or +i b +pro mis +produc ts +moun tain +flori da +regi ster +tre at +rec ent +fe male +boo th +mat t +ve hic +s op +mo tor +suppor ting +phi c +ex tre +dr ink +lan e +th ird +p s +con stru +ce re +far m +ðŁİ ī +tu red +ðŁij ī +c ats +a j +gi e +shoo ting +as ked +paki stan +am e +m b +g il +leg al +squ are +in vol +dra w +oo oo +!! !! +opportun ity +p y +e i +b ts +teach er +charac ter +john son +br on +ly wood +ch ine +c ing +c ine +d ge +gam ing +russi a +ci a +quo te +ric h +go v +flow ers +sp iri +st in +grow th +ðŁı ¼ +comm er +j uni +mu m +r an +s na +a ren +c b +ac tor +col or +si t +pa ir +ch i +bo w +acade my +hel d +r ang +me tal +y l +ac tive +probab ly +t ch +need ed +spe e +cho ice +ital y +ry an +ðŁĩ º +flow er +v it +m n +found ation +b ak +si ons +ne igh +f loo +he ard +re mo +fre sh +ing ing +re f +to wn +cl ou +je sus +spiri t +cou ldn +z es +ðŁĴ Ļ +willi ams +pro ce +moder n +pro cess +sho es +cre ated +tri c +issu es +ann e +att en +de but +h r +n it +sti g +a po +e ps +z u +ã Ģ +si x +car ds +lan gu +fam ous +tour nament +se l +e bay +y n +st on +k ick +announ ced +k am +vo c +brilli ant +hou se +che ese +war ri +mus ic +ho ckey +ðŁĺĤ ðŁĺĤ +sk ills +au tom +smar t +med ical +mon y +e x +gu ar +gi ve +pers onal +ven tion +al li +pre ss +flo or +m c +victor y +hi m +simp le +th or +ðŁĩº ðŁĩ +ta il +lu cky +ale x +qu ite +bo t +ssi ons +chall eng +c ann +amaz on +h ell +b ought +) : +ed y +secre t +produc tion +inde pend +de fe +ad ded +p r +p ag +be d +gre atest +with in +j ay +ðŁ ¥ +ire land +re ly +s d +te xt +dri ving +pro gram +spe ed +col um +str on +à © +fore st +â ĸ +mach ine +co in +sc ar +oun t +bi e +¡ ï¸ı +por tra +comm on +wre st +recei ved +kno w +inve st +pl ans +ac cor +ad op +ter y +re ali +p p +k al +art work +me an +go d +inste ad +an ci +motiv ation +as ing +inspir ation +up coming +polit ical +euro pe +m ers +heav y +ðŁij į +fe bru +scot land +ou gh +b t +bo ss +sche du +spe ak +n ick +u red +in o +e k +ri sk +tor y +pres ents +b on +ru g +st ates +exhib ition +il o +m ill +br ought +: -) +tou ri +com e +offici ally +champi ons +do ors +re p +po se +ex tra +k ings +soc cer +squ ad +app lic +at a +some times +t ari +excell ent +ðŁĺ ĺ +stra ight +car ol +ri p +âĢ į +gra phic +m ol +elec tion +febru ary +as ons +l i +di r +m t +n ick +u su +m rs +com ics +inst itu +cor por +v i +ðŁĻ ı +tu ral +di se +ac ci +we are +am ong +sho pping +t ill +wh at +cha ir +sp an +chine se +innov ation +jo y +k it +cent ury +ob ama +ph ili +f c +re ach +c iti +ul ous +n on +d ang +happ ening +bur n +p el +or ange +d v +k ick +cla im +ing ham +ph y +no v +pod cast +wh i +ni ghts +ear lier +be ar +la h +exc iting +or a +gi ven +s lo +memor ies +contin ues +produc t +gh o +c d +kno ws +ðŁİ ī +publi shed +discu ss +y ard +i phone +tri es +w all +fe b +are n +tru th +win ners +tu re +diti onal +milit ary +proble m +m and +do g +lo ss +c ric +can adi +ve ter +villa ge +" , +y r +un g +don ald +ag ing +bir ds +sci enti +le s +th is +regi on +tic al +itt en +il a +ðŁĺ İ +d ad +di am +abo ve +st ren +li t +p ir +la b +fo cus +bus y +d ur +app ly +s ma +auth or +ac i +exe cu +dom in +re la +jack son +at o +wash ington +ðŁĻ Į +k ill +popu lar +ce ment +ro ad +e ating +loc ation +v ent +ar re +n an +cu sto +advent ure +or din +spor t +ul t +lo ck +questi on +dri ver +land sc +on i +k ins +p d +jor dan +te red +k k +a f +chil d +s p +just in +en i +s elling +z o +wh it +bo ston +partic ip +sig ning +happ ened +he at +m am +dre ams +lo ws +gra ph +the day +head ing +br o +ble ssed +vi c +ve gas +h d +in ning +ro man +and ro +den ti +u se +c it +pro gress +writ er +bo b +ff s +gro wing +b ly +aw are +ex am +sp ent +be t +sc ore +bey ond +do cu +ad el +s f +cou ra +colla bor +in c +priv ate +bo at +* * +z one +p ha +b ill +to tal +plan ning +to wards +plac es +pre view +cre ative +dam n +ide as +se ems +po ten +say ing +di splay +s w +a qu +lou is +by e +li l +e mail +we stern +ger many +ell er +re s +f ant +ment ary +de als +ric hard +jer sey +stren g +ra d +pizz a +mon d +w are +l ac +g i +ar chi +c d +yel low +rec ently +re ach +à ¹ +kitch en +desig ned +tr y +g al +restaur ant +at ure +w w +j as +l ma +ðŁij Į +pa in +av o +min ute +sch ol +ther ap +tic ket +d ry +jap an +diti ons +ter ri +sel ves +happ en +t up +ma g +cop y +sh er +free dom +f ile +speci ally +tor onto +lo ad +g ary +re y +answ er +lo y +cau ght +pri ze +u ne +fic ation +ni ger +sy d +tou ch +feat ure +jaz z +recor ds +him self +di sh +ro ber +spot ted +ma ster +wa ve +fin als +bu ll +for um +al d +re comm +ch a +a e +d oo +inst ru +tru ly +l g +in k +bro thers +de st +j im +m it +clo sed +is on +tri ed +s anta +af fe +w an +hor se +g row +camp us +rel ation +nati ve +jour n +go v +o ct +k it +b ound +part ner +re ma +crow d +! ) +c alls +ra il +qu ali +solu tion +con test +con vers +sn ap +b ase +in iti +ta x +y e +ent repre +it or +constru ction +foo d +present ed +n ings +cli mate +k m +mo del +b j +blo ck +present ation +dre am +fi x +c alling +bus ine +con gress +under stand +we b +val ue +ï¸ı âĥ£ +mex ico +it ely +ki m +char ity +ref lec +bl an +fl ying +anal y +famil ies +b and +reci pe +celebr ation +ac cep +ar y +to t +g b +intere sted +cap tain +âĻ ¥ +ti p +ab sol +bra z +inve stig +o logy +de c +tru ck +ver ing +c lear +don t +go tta +ad vis +beg ins +ma ss +de scri +blo ck +k im +davi d +son gs +memor ial +feat ures +su stain +' . +gra b +jo se +v a +con serv +se ts +man chester +fi ghting +de gre +ag a +in d +sle ep +pos ition +ha ir +sig ns +pol icy +it o +al ert +st am +sp end +w y +absol ut +d m +anim al +my ster +success ful +proble ms +ro bo +k ay +gar den +p d +may or +d ale +t ol +off ers +vis iting +friend ly +tre es +offic er +accoun t +ke vin +ðŁij į +gi ant +contin u +con su +tr act +n fl +ðŁĺ Ĭ +h q +b ility +a ar +dis ney +te en +on ed +wh ite +tra iler +de dic +al one +absolut ely +dig ital +willi am +in ation +s wa +e e +enti re +ger man +ro ll +h its +co st +st ay +th a +ali ve +accor ding +co t +liter ally +her it +re ti +haha ha +exper i +li kes +g t +ste el +__ __ +ch air +christi an +to wer +diffe rence +m d +tre ss +mi d +prin ce +afric an +fe der +foo t +car ri +ser ved +r ice +sh all +feat ured +ck er +rec ru +po e +sen se +ni fic +com edy +cont ent +f at +po sted +con tribu +tim ate +li ver +mb le +inter net +ag e +europe an +cl ing +gla d +ff ic +sc o +ak es +el le +ter min +ton y +p ale +col our +seri ous +pat ri +movi es +b m +professi onal +ad o +al u +br inging +f alls +isra el +ter m +langu age +bro ok +man n +commun ic +can not +ac ti +p he +y an +entrepre ne +tur key +log ical +lon g +ar m +ur s +work ers +ing ly +gg s +ri c +tu al +recei ve +op ens +ge ar +soci al +fe et +c king +ad ver +fin an +fe els +sp la +h r +ea ster +bra in +ã ģ +fi g +le dge +ne arly +prote ct +ma ssive +e th +aw a +ðŁĺ ģ +y rs +aware ness +defin itely +k n +imag ine +k u +syste ms +ðŁij ı +f as +li k +provi de +am o +disco ver +inf lu +ma ker +g az +fit ness +stre et +er s +te d +w c +ys is +pos itive +hel ped +que st +andre w +bra d +b in +hang ing +l ing +bri ght +se ction +ma ss +ðŁĻ Į +follow ers +ho sting +tem por +fla g +a ve +let ter +k ur +re qui +of ten +cry p +su ff +âļ ½ +russi an +treat ment +al le +ha y +l an +keep ing +hol y +power ful +pre dic +fun d +e specially +windo w +je wel +il y +ðŁĴ ľ +gener ation +app a +seri ously +o d +ðŁĺĤðŁĺĤ ðŁĺĤ +cer ti +iri sh +ðŁij Į +mi ami +be th +v ity +se cu +che f +cri me +graph y +ma x +arti sts +re volu +gu ard +spee ch +u c +upd ates +fac es +st ant +chang ed +repor ts +low er +pe ar +n c +k il +loo ked +spe aker +s f +re spect +ok ay +oce an +s itting +architec ture +tra il +se at +i ra +le g +japan ese +d am +u lar +sw im +polit ics +finan cial +ol d +mou th +at temp +de stin +fi shing +atten tion +me m +chang es +deci ded +reli gi +g in +c av +z z +ad am +ma c +wr ite +beg in +sc ul +al ter +is s +ath on +imag es +m oo +jo ined +ðŁĺ ī +âŀ ¡ï¸ı +pas sed +mu sli +h ir +lar gest +cam er +com ic +gh ted +rug by +bur gh +gg ing +te sting +pre par +lau gh +al ed +impro ve +beli ev +adv ice +sha res +he art +tur ning +s b +t el +caf e +n es +dani el +pat ter +t z +se tt +par k +c and +st ick +happ ens +bri an +ne west +e pic +ad or +ki es +war ning +anim als +custo m +ar c +di an +gol d +cor e +t f +c ity +pan ts +re ality +con fi +in ju +fo x +gu il +k new +âĺ º +cor rec +itu de +d den +. # +re duc +pas s +f on +y a +ow ner +re turns +n c +e ast +ap ol +in sur +th o +si m +juni or +be e +ang el +att le +elec tric +hor ror +cra sh +e ye +pat h +sou thern +emplo ye +ge o +t an +ha z +r ally +ðŁı » +proper ty +was n +enjo yed +gre y +g as +bre w +nor thern +hol ding +g p +ta ke +ch art +ly n +dr ama +z o +pa id +throw back +cu p +discu ssion +down town +w ill +le w +b is +t ary +bre ad +up on +r ate +teach ers +it ation +anc ed +cy cle +choo se +d c +ir an +co w +da ve +ra ise +prin cess +fa ith +- > +indu stri +sp ain +guit ar +fac ts +m n +sp en +cour te +go tt +projec ts +au di +o sc +pe ter +s and +intere st +happ iness +ven ue +sol di +surpri se +poten tial +per io +custom er +i i +g ni +manu fac +e co +bro ken +sing er +vel s +wal es +hu s +in j +f our +tal ent +d ying +mat the +fil m +jo ining +s ell +j ar +lma o +sur ger +bb c +sour ces +au stin +ni k +char les +f am +prin ci +ange l +cas h +lo t +o red +pla ys +pl ate +don e +memor y +br ings +n ba +solu tions +teach ing +gr ace +cir cu +hel ps +foun der +mar y +expl ore +de cor +par ts +ch o +inte gr +ha u +is es +pu tting +in er +r it +v y +mic hel +blu es +every day +for ms +bi o +ye ar +p in +t ter +spr ing +) ) +po t +al ing +perform ing +sh an +plan et +mus ical +head s +it alian +stru gg +âĢį âĻ +w ings +pu mp +h h +tr ou +a id +pri me +ear th +pa int +mon t +am y +bb c +fab ulous +fru it +andro id +bour ne +cere mony +enti al +? ? +deb ate +on ing +dra ft +sol ar +t x +j am +cor n +!! !!! +bro o +mil k +po sed +o hi +mo vement +b ren +part ner +p g +et te +ar ies +sh out +n g +leav ing +t ells +sen s +ta ste +kel ly +wor l +gy m +ric h +e gy +pi d +ma s +â Ĥ +courte sy +fran k +incre ase +wr itten +pp ers +re l +ha i +s as +s ound +tt i +w ich +ri ver +.. ." +a g +fel low +ro me +sm all +gen cy +ic an +lux ury +pro of +me t +wild life +mom ents +ra ther +cor ner +com pe +canadi an +lik ely +therap y +li am +econom ic +indi e +rou te +fi ght +ho pe +se tting +ant ly +cro ss +fant asy +de e +sket ch +comp li +ym i +ru les +engine ering +fig ure +ro w +. , +f w +syd ney +w ou +t ation +dre w +us es +the re +sp read +struc ture +pat rick +appa rently +ro s +h ills +w we +ann y +com mission +di v +f ying +con sul +anal ysis +ex i +ten nis +vehic le +ðŁĺŃ ðŁĺŃ +as s +high ly +op ened +b ann +ðŁĴ Ļ +mp h +wi shing +v or +fi f +give away +r r +ra y +je ss +g at +ic ymi +x it +high est +yor k +pi e +invol ved +high er +ri e +mal ay +int elli +desp ite +che e +sar ah +be an +reco gni +ar sen +tal ented +pas sion +ic h +ab c +lead s +dise ase +v is +se c +pre senting +m illi +hol e +sho ts +de part +surger y +gov t +b in +du al +e vi +lon ger +ev ol +scre en +portra it +et c +lo se +ch at +p en +p i +om a +s ick +er c +compan ies +en try +plan e +gr y +ven e +liver pool +premi ere +sha red +a red +fil ms +ir a +holi days +cric ket +ici an +v ing +. ) +ul timate +di vision +con duc +se pt +for ces +mon t +s mart +disa pp +sun shine +in d +b less +ma de +col ors +fran k +ir on +bott le +s go +m ood +j ason +er ic +bir th +te en +respon se +tar get +state ment +fe ar +th el +al um +ar ab +bl in +direc tion +ste ps +er ial +wor ked +at l +ðŁĴ ķ +fel t +pol i +scen es +hom es +b ell +e at +ate ful +t in +l ace +fol ks +p se +an n +wis dom +fa v +but ter +s r +are as +sm oo +bi z +dg es +app o +mo re +the m +effe ct +windo ws +sun ny +cap ital +tot ally +c ities +gr ant +mb ers +s low +au tu +il ities +w ro +ri sing +st ics +viol ence +i gh +qu ot +h it +t c +herit age +bu ff +ne s +z ar +den tial +ex ac +ed ge +de ep +aren a +be came +benef its +mar ks +mb er +a z +am es +pre ci +dra gon +re g +d ings +do s +ðŁĴ ª +n el +s ity +me al +di st +leg end +pur chase +pic al +st ick +f at +du ba +profe ss +car to +pro f +coun tries +respon si +se qu +fa b +tribu te +hon ored +prac tic +pur ple +an ton +pa red +t ough +summ er +environ ment +s ons +ðŁĻ ı +m ps +gi es +her oes +t elling +hen ry +f en +know ledge +Ģ ï¸ı +f r +ne g +u re +ac king +hear ts +s oo +hol lywood +ju mp +sau ce +schedu le +tur n +yo ga +cre ating +c ket +cre ek +â Ń +custom ers +ma dri +gu l +asse mb +moun t +c ell +to p +st al +dav is +t wi +sig n +premi er +iti ons +he aring +un k +pati ents +app ear +heav en +al ty +doc tor +a e +plat form +je ff +ðŁĵ · +regi onal +bi d +box ing +ex ten +or ity +a w +w ise +il le +sever al +bi e +s itu +sy ria +âľ ħ +remin der +enter tain +li on +part ners +in n +ph ar +f au +pl s +expe cted +sug ar +deci sion +s b +ch ron +associ ation +leav es +vis ited +sh ap +ðŁĴ ĸ +fur ther +h ann +w i +run s +l er +fun ding +fil led +.. .... +tin y +han g +or g +co ol +se min +ðŁı Ĩ +spon s +nav y +sa int +dru g +d al +r oun +co vered +tra ditional +invest ment +de te +al ism +f low +n is +sun rise +fe at +f ted +we ird +je re +ve gan +medic ine +an o +ac cu +deli very +temp le +chang ing +wil son +phili pp +re fe +n d +is er +g ay +r and +ati ves +t ely +p and +intelli g +g are +am bas +de mon +commit tee +strate gy +refu ge +bud get +prote c +pi er +ex press +nom in +econom y +al low +ic on +gal ax +o h +indi vi +dem and +vir gin +lu ke +ali sts +man i +s mi +ju dge +ent y +mic hi +resul t +am ed +spe aks +' , +hou ston +sh in +b ing +fl y +ch em +au to +v as +ge t +ar m +thank s +d in +gan g +x x +si on +loc ated +p l +jo sh +in fo +jo ins +adver ti +ot d +el d +si e +re asons +v ent +ðŁĩºðŁĩ ¸ +â ł +convers ation +stu di +ðŁĶ¥ ðŁĶ¥ +go s +s ounds +un it +mu sc +ge l +ack ed +pac i +co s +de re +u u +a o +la m +inspir ing +ar ms +tw are +mat ters +ad dic +du de +ex t +cri sis +b ath +me et +sing h +expe ct +del hi +resc ue +wor st +au g +shi pping +ser ving +st o +dar k +ac es +histor ic +landsc ape +desig ner +b illion +gr ateful +wa ke +e ve +m iller +hou sing +dy nam +is co +be ha +sh op +pr ou +e as +a sia +e ding +k on +depart ment +aw ar +mar ine +in ci +photograph er +ta pe +lo go +r ings +d it +-- -- +vin yl +w c +vo ting +se ven +ambas sad +dal las +t u +com ment +k ra +b les +w ag +u d +au dio +stri ke +offici al +o ts +me tho +to ols +ra di +al an +hun t +wat ched +a ke +fa ke +drin king +mer ry +m l +b day +ri o +ni ke +c ant +re pe +co stu +mur der +ak ers +ch ers +ou ts +beg inning +so s +ad es +n in +not es +wro te +sol o +c i +li ghting +ur ban +bre xit +att end +shir ts +pla yo +ac tress +pl ic +stand ard +quot es +par ade +anci ent + © +tur ing +re e +pri mary +fla sh +citi z +mat es +ste in +z i +clin ton +sk in +gen e +hu m +g ar +t le +y i +fo cu +de an +pl ants +cy ber +b u +om e +ho p +ad dress +ti x +gi fts +relation ship +sub scri +fe ed +exac tly +haw ks +ex o +stre ss +s n +arre sted +an e +sof tware +z ero +the me +mu mb +im migr +mi a +make up +ple asure +uni vers +har b +eng ine +ap er +r in +br a +institu te +le ather +al th +sing ing +co s +gh ty +me as +st ic +si de +insur ance +co t +pit ch +moun tains +cri min +su pre +valent ine +at er +wou ldn +sc ale +rel ated +re gar +star tup +pack ed +mi ke +week ly +p ts +coun t +ha r +gott en +min d +ber lin +con ditions +swit ch +cor n +sa ve +g li +emer gency +tun ed +sto ck +discu ssing +every body +s day +whe ther +wrest ling +ec es +gen der +ch en +ðŁij Ģ +madri d +mar athon +e gg +i er +th x +as king +kore a +wol f +ay a +g m +g au +at ory +v r +gra ss +k illing +b ble +ur o +un i +e th +sh ore +th en +re ale +bot tom +ex erc +k ar +or ies +ad ri +san ds +se x +. ' +volunte ers +per form +par liam +inclu de +deli ghted +execu tive +fu el +kis s +ã ħ +char ge +h u +ca kes +ve t +g lu +agre e +pr ices +n au +h l +g ru +ra j +streng th +b ic +sp ending +al es +av en +b last +: ( +yo f +nor mal +si x +qu ick +se a +d aw +mee ts +lo vers +upd ated +po tat +comple ted +coo k +opportun ities +p ure +organ ic +tem per +c am +avo id +par king +duba i +and o +di stri +to y +comple tely +don ald +tri al +bas s +b oun +back ground +v as +mar vel +lu m +ru s +t ool +com missi +throw back +fin ding +is lam +! ? +st op +e vil +or al +resi dents +i denti +o ak +ðŁİ ¶ +l il +span ish +chap ter +sto pped +direc t +ho sted +pic ked +lab our +lew is +defen se +à ® +health care +wh is +mat h +pe ak +ra ised +fi x +bu ll +th ir +chel sea +fol k +tr e +can di +pau l +ei ther +ad am +poe try +jewel ry +ðŁ ¦ +pr ay +Ø § +g c +o z +wi shes +fore ign +sun g +lear ned +en e +n ing +micha el +illu stration +legend ary +w av +b au +ðŁļ ¨ +cal end +stre ets +â Ĩ +mon ster +bu ck +g r +scho ol +ba th +wa ste +ne ck +ha wa +be ach +re plac +jec t +on er +fac tory +coun t +ðŁĵ ¸ +mor gan +der ing +se an +steph en +de p +no vel +vide os +ic al +press ure +arsen al +ex pre +ir s +tren ding +ss a +fla sh +re sear +thr ough +profess or +scul p +to s +gg ed +mm a +be e +a pe +hun ter +am i +he i +pla stic +bu cks +uni verse +le gen +niger ia +ple ased +ri s +thin ks +autu mn +i ds +d is +anth ony +ðŁı ½ +ak ed +gla sses +fin ance +z er +k as +con tract +nu mbers +sh aw +partner ship +t il +laun ched +s al +victor ia +theat er +usu al +nam es +perio d +eli za +i th +bar cel +ro cks +bag s +mat e +distri bu +j on +di ffic +ali zed +cur ren +sco red +b ha +du blin +ro se +in ted +soli d +beha vi +wal ker +simp ly +garden s +head ed +in i +ohi o +we ap +f o +gl en +e state +ran dom +th under +thr u +k ill +jac ket +it i +entertain ment +thanks giving +ent al +en coura +el o +a ther +tan k +high lights +f ting +ru le +model s +bor der +bj p +hus band +in done +ken ya +be ars +al o +n inten +pi x +str o +or ders +sal ad +ro ads +n or +l ation +sop hi +ðŁı ¼ +pi eces +b one +min s +inclu des +nu tr +phi l +s ent +fun dra +ga in +bor ough +n ad +mon day +activ ity +it ems +be coming +ken ne +de tro +car di +gue sts +u x +world wide +sever e +new s +thank ful +fic tion +ve ge +m all +si an +er al +inj ury +le e +men u +danc ing +scot ti +exam ple +( # +na i +studi os +ba i +ðŁĴ Ľ +j av +diam ond +vin ce +ric k +prote ction +lin col +cham ps +appro ach +d ar +m ile +clou ds +je ff +in fin +l ers +p les +pe ace +go p +âĻ ¡ +tech n +str a +a verage +ef fort +introduc ing +di versity +austr alian +am p +boo st +s ke +pati ent +appreci ate +ici ans +pu r +f ell +woo ds +illu str +ðŁ ĸ +ag ency +ac tions +brit ain +under way +se attle +el and +ag o +f ill +stre aming +pro test +challeng es +ky o +et sy +coo king +exper t +ru ss +rain bow +commer cial +sp in +be ats +c ry +val u +el i +th row +gr ams +le vels +michi gan +c ad +ador able +const itu +w s +pu b +mid night +th at +net fli +braz il +die go +regu lar +jo y +âĤ ¬ +li qu +ea stern +k ni +fl at +n p +bro wn +w er +se y +tt ers +ac ting +v anc +cy cling +program me +ra w +comple x +tat too +throwback thursday +se ssions +ro oms +si ght +speci es +bom b +lau gh +ke eps +mo on +offic ers +con ver +t r +ha sh +t ack +ri ous +ad ap +a j +reco gn +ex po +sug ge +confir med +rol ling +dre ssing +ic t +fri day +ph ones +ri dge +con cept +ro y +ke ys +ef for +c ate +k ne +ev en +l ay +commun ities +mo d +n az +every where +al ab +bit coin +ban ks +out door +feder al +sto res +h p +c al +m ely +sig nific +be ar +re public +clo ser +al lah +pic k +x d +pal ace +ch ill +b am +er ous +un a +al len +out standing +olym pic +supp ly +fi gu +v au +l p +char lie +un es +> >> +legen ds +ici al +co ast +benef it +mul ti +f its +far mers +am ount +si sters +har ve +hon ey +que en +b ers +pl ann +âŃ IJ +m u +barcel ona +al ber +stat us +re main +ex tra +c andy +vi ous +âľ Į +o v +warri ors +-- > +ju mp +am ar +x mas +stu dies +i ors +k or +don ate +pre p +fi sh +im a +pain ted +ad mini +co splay +spor ts +dro ps +fi ghter +evi dence +ðŁĴ ª +la ke +ro b +cine ma +pro file +à ± +stan ds +leg acy +sh ape +ro of +ci vil +i ans +sy l +sh am +vo ted +re tail +ph illi +li sted +du ty +n b +th es +f are +au ction +ffici al +stor ms +d p +l oun +sh ops +al y +ani me +multi ple +ðŁĺį ðŁĺį +psy cho +je an +ap art +candi date +gg y +con f +jose ph +w ick +me at +fr ame +c l +for got +ph y +f ing +li ed +re p +se ed +f all +u fc +nu t +lin d +mo de +fiel ds +en ce +s ley +ðŁ¤ Ķ +ch ill +follow ed +announ ces +cor ru +tro phy +them selves +ac le +al du +k ong +l on +s v +bro ke +ander son +ta i +stor y +tempor ary +activ ities +k ati +ari z +cry stal +spo ke +extre mely +tra ding +ðŁĴ ļ +à ¼ +in ch +ed in +out fit +equ ip +ma di +form ed +be ef +po p +ti ger +this day +ti red +neigh b +re tro +is a +un t +t as +kan sas +de st +secon ds +ta y +hur ric +o u +galax y +dad dy +bro w +bur ger +en ced +de sk +ac cur +secre tary +el ite +k ab +ch in +touri sm +bud dy +ici de +dre ssed +u d +vac ation +che ers +com for +charac ters +j et +bu ying +l ins +n ap +reale state +li e +af c +i ii +f ame +n r +b at +ag ent +ma kers +âĢ ¼ +sec tor +op ti +le on +di et +pra yer +hi p +mi r +le x +br y +an a +pas sing +w en +reco very +ak i +po pul +res ort +mar ia +stu ck +read s +ti er +perfe c +netfli x +p oo +cham p +o c +re duce +we red +comm ents +cla im +acci dent +s ag +h ack +sal t +kin da +k iller +i os +z y +ex change +lec ture +eng er +ic king +t au +reve als +pri son +z om +gh an +u l +jour nal +i ot +tr in +jon a +govern or +cap e +quar ter +spec tive +impre ssive +bab ies +t x +m ill +o y +har ri +jo int +su e +collabor ation +tren d +revolu tion +re new +alum ni +ge tt +sh ell +sun day +ent u +ni c +donald trump +block chain +paci fic +expla ins +sp y +ad voc +par adi +to f +star ring +p av +fe ed +br ac +smo ke +ham p +y am +to kyo +si mon +d h +e ffici +phys ical +n j +ell i +s low +gradu ate +americ ans +ti fy +f red +ap ore +fin ds +rob in +we t +not ice +se mi +un ve +k om +pil ot +scre ening +da ily +ðŁĴ Ĺ +roy al +sp a +vo tes +n ag +wh ate +att ending +exper im +ad dition +k ate +sto l +m ali +foo t +chri st +ch an +de e +lic en +glo bal +mo ore +ti a +bri gh +myster y +y ay +âĿ¤ï¸ı âĿ¤ï¸ı +cre ati +me chan +clo ck +di c +âĢ Ķ +pp er +al ph +through out +al low +re sources +selec tion +ham il +bb q +aa aa +virgin ia +dis ney +en g +so red +drin ks +f ancy +consi der +end a +jan e +hand made +du l +on tari +i us +s ville +color ado +whate ver +whe el +promis e +ne ver +desig ns +ab ly +sex ual +vanc ou +at i +con vention +cul tural +sing apore +pro mo +load ed +gla sgo +pp l +n oo +ke e +ste m +men tion +i do +cru ise +ri ding +be comes +be y +âļ½ ï¸ı +tw in +dedic ated +na sh +de si +work out +jen ni +i v +grou ps +rela x +pho eni +li ft +mix ed +m ck +p c +mu st +me tro +ci es +y ar +a im +ang er +i e +rec y +marri ed +dro pped +eng ag +le st +ambassad or +op h +de s +w ick +assi stant +nat ur +fa il +l td +shor t +k ap +sha w +bi gger +rema ins +crit ical +sur vey +co verage +er son +win d +n b +bil ly +let es +ac ts +jim my +at lan +al and +t c +import ance +dam age +f g +stor age +tw t +bon d +bal ance +cr ying +pu ppy +vo te +pu sh +ðŁĴ ľ +pol y +me l +lon don +terr ori +effec tive +corpor ate +atl anta +jac o +nas a +gre ek +sen ate +i sh +ev a +intellig ence +effor ts +al co +k un +h all +di ag +claim s +fir st +h b +ba e +v ul +pu ll + ° +se par +spe ed +vic ti +on thisday +audi ence +r ates +te ach +fil ming +bu sh +son g +y um +br un +ra ine +aw a +par ks +ð Ŀ +ra bb +ra ch +ra id +reach ed +ra il +mo ves +selec ted +fr i +ra ising +om y +st ones +su k +franc isco +cas es +cap it +con fu +w tf +po ke +equip ment +gre g +ess ential +off ering +ne x +pi es +be c +cre ation +chair man +cro wn +w al +john ny +shi ft +ne ck +ban g +bir d +ðŁĺ ı +du ck +re serve +de pu +ma sters +over all +no tic +ju ice +sne ak +che er +cla sses +eag les +n ca +car pet +ci vil +coach es +har ris +u ps +b alls +dec or +mar tin +ro s +v ice +announ cement +who se +ti gers +ste red +c ts +dr am +ste el +youn g +inst all +supp o +recor ding +de ck +se ats +l der +ang le +bo t +sty les +elec tions +for tun +n ab +but ter +ari an +ka sh +in ner +ou red +be ast +we i +ic onic +exper ts +ne cess +b eng +jam es +li a +gre ece +ðŁĵ · +ðŁĺ ģ +good bye +m itch +tw ice +mumb ai +ste am +ru sh +med al +ne tt +fashi on +t ar +r s +sav ing +ric ul +l m +sleep ing +brook lyn +mis s +sen ding +disco vered +sp here +of theday +k icks +missi ons +w right +er n +ght ly +i ous +mel bourne +star tu +mo ved +car ry +d ak +ag ues +bel gi +e ma +way ne +do t +er ie +pe l +it unes +matthe w +no body +est ab +cal m +win ds +lu c +prep are +tren ds +exerc ise +adv ant +ðŁĴ ¯ +athle tics +app s +c tions +adv ance +laun ches +litt le +real donaldtrump +eliza beth +carol ina +hu b +hi dden +n w +us er +pol l +great er +mo st +f ed +p at +life style +s ati +sco res +marri age +l r +aven ue +de serve +ri f +ðŁ Ĺ +wat ch +champion ships +gr ay +en ni +cot ton +g om +whe re +pack age +su m +ab solu +new ly +foo ds +ty ler +assemb ly +musli m +ban k +re memb +op tions +produc er +land o +fun ds +u pper +shad ow +pro gre +co p +ing e +leg s +detro it +hill ary +jo se +gi ants +sou p +sustain able +t us +clo thes +roc king +n z +min ne +mat eri +bru ce +ear t +ca sting +independ ent +thou sands +ta h +de cl +veter ans +li ons +wra p +âĢ ¦ +de ss +bl ing +st ine +e ggs +o on +clo sing +z ay +at t +bac on +fa il +ariz ona +de pre +gho st +new sp +w ers +vi p +li ked +id ent +volunte er +ad ult +pu pp +cir cle +mat erial +degre e +gro wn +boo m +calend ar +su r +vie wing +ath letes +ch and +re ll +asi an +en tr +vol ley +victi ms +bo dy +m ama +trans fer +ge ek +in dic +sav ed +ma i +g ent +it s +loun ge +k ol +the ory +situ ation +is lands +ar th +z oo +floo d +vi ously +show ed +parliam ent +ch ev +el ine +at trac +ab ad +ta il +h rs +lu s +por tu +gor y +provi des +to ys +de ath +in fe +an ce +g le +li am +lo ver +hu d +dv d +reve aled +g w +re ment +ca the +l ying +ra dio +der by +stor s +che mi +hosp it +âľ ¨ +' : +ilo ve +le mon +re public +s ni +ne ss +do or +re action +pre gn +fla v +schol ar +spo tify +is ation +vis ual +aw are +spon sored +jo ke +less ons +leg is +lo ck +si mil +ðŁĺ ĭ +kin d +la y +ma h +ho ping +vancou ver +as er +clean ing +gal a +thre at +la p +ach e +ro mance +ex pen +re post +z am +e pi +mir ror +o ak +ad ul +bat man +s lu +l c +vie wed +re views +d ates +indone sia +acti vi +off en +lea f +i si +ag ricul +costu me +s ites +spir itu +appear ance +ir y +st air +applic ation +spec tac +ic ity +ski es +hand le +pun k +paradi se +t n +de al +provi ding +do c +recei ving +bre w +micro soft +à ¶ +fer r +me tro +th ail +y um +car ter +à ¡ +gent le +bre aks +coo per +show case +cu tting +egy pt +bab y +semin ar +gl ori +ss on +fa ve +re hear +lo tte +la dy +al as +pre p +deli vered +nu clear +ir o +engag ement +at ta +con ven +z an +gl ory +hol ds +busine sses +str ange +sch e +it self +gra d +mar kets +f alling +st ats +ge on +bu dd +li s +she et +thi si +co lo +deser t +regi stration +ig n +expla in +inter ior +la ws +writ ers +spr ings +k r +fri ed +blo om +inf ra +a o +cre d +pa st +line up +bo o +bre a +boo ts +celebr ity +att acks +bro ok +ev es +ex cu +cher ry +oo p +fas cin +boy friend +se as +n ine +effec ts +po wered +k ha +ðŁĺ Ģ +sh out +con dition +i j +her o +enter pri +win ter +applic ations +sho e +g el +batt le +pro grams +w art +ðŁĴ ¥ +ra p +ho l +dang erous +di a +coun ter +ric s +i or +k night +co at +emo tional +at ures +d as +whe el +fore cast +tran sport +glasgo w +king dom +prepar ing +im medi +ff in +awar ded +prin ting +ro man +fight ers +any more +bel t +p ine +win e +x i +employe es +logi es +al led +de mo +birth day +ange les +lo g +dri vers +neck lace +k ath +s it +athle te +ef s +s burg +pur pose +resi stance +rele ases +t is +vari ous +deli ver +ch al +s anc +opp o +cra w +neu ro +dr a +suppor ters +sna p +diffic ult +swe ar +logi st +pa th +attemp t +à ¥ +swim ming +ste ve +hur t +inclu ded +b ap +wa re +ðŁĴ ĭ +end ers +ja ke +le eds +cli mb +l b +im ple +li sa +clo thing +ðŁĺ İ +d t +com pla +sw ing +stra w +v als +k le +us ers +stor m +cu ts +ontari o +p an +hand some +i ow +ar gu +chec king +scotti sh +Ķ ï¸ı +si er +em ma +po d +patter n +de sh +en h +ed ward +t ing +k h +hal f +lincol n +mo ther +al leg +r c +volley ball +d n +g ay +all y +le ton +gro ve +l oud +adv anced +re spec +cli ent +supre me +thail and +ho w +gi g +to i +do t +dol lar +ðŁij ĩ +p it +r b +h n +produc ed +gg ers +âĨ Ĵ +ml b +can vas +fin eart +us d +in the +p son +actu al +s l +t b +ip ad +en sure +u mb +w d +sk a +mar s +k end +f eli +th ing +count down +absolu te +r out +dra l +p y +inju red +min t +hun ting +mm er +s age +li gh +ac ity +ex pan +mur ray +ar o +sec ure +four th +eag le +reli ef +st akes +industri al +clar k +under standing +see m +pl enty +sil ver +cla u +thre at +sa il +pro duce +ab str +is is +b r +eng ers +wor ry +bie ber +s j +just in +reali ze +ky le +esp n +fil ter +s ch +ty pes +game dev +d ing +twit ter +soldi ers +p om +car bon +y ards +child hood +ri ed +ke l +ele ph +t ons +key note +qui et +wi re +po sting +is sa +repre senting +bac ks +alex ander +celebr ates +ta ining +| | +ch or +esc ape +pe ek +ti ves +fiel d +ssi e +im pac +spons or +r c +we dd +cann ab +si des +trac ks +com par +con trac +techn ical +bi ble +expl oring +sh are +tra v +n ate +ill o +sc ru +m ingham +gun s +of the +sh ame +se es +ca tho +ac cess +ce l +repor ted + » +mari o +p ad +hope fully +ou se +y on +disapp o +ol o +p itt +pa c +ga p +cru sh +s g +k le +ge m +emp ire +dir ty +a is +avi ation +ze aland +fac ing +high way +d anny +spi der +ot ta +ðŁĺ Ħ +w y +col ours +in fl +co sts +olym pics +au s +h m +ho ward +pas ses +lau ren +mu sh +op in +r ho +disc ount +oper ation +em ily +mm m +cham ber +d il +to yo +shi p +sam u +pic tured +un ic +po l +keep er +carto on +st en +ig nor +n ations +n l +ta sting +deta il +offici als +mo tor +franc is +ed itor +ðŁij ĩ +pe ts +rang ers +t g +r n +w ri +nic hol +i se +spo ts +ani e +chec k +tri ple +ku mar +spe akers +ic ing +pre pared +ab use +friend ship +mon th +swi m +air e +sc ent +hamil ton +indi an +j es +yum my +te ars +da wn +i zed +worl ds +ðŁ ķ +b illi +st one +n hs +ba sic +p or +st le +ir on +ol der +cle vel +e ing +ðŁĺįðŁĺį ðŁĺį +prin ts +fir m +air craft +fin est +devel op +aar on +t z +gra ham +own ers +fo li +less on +qu es +bab e +cra ft +ph en +ju n +bir mingham +v ine +ll er +i an +fineart america +evol u +st ab +im per +war d +com ic +wi z +inv ited +du ke +mat ch +por ts +ro ger +diag no +ke pt +te st +vis u +r hy +so c +to x +b aker +sur face +co vers +man s +b its +x box +ff le +n an +gar d +h art +wat ers +v illa +re tro +light ning +catho lic +democr acy +neigh bor +pen n +cr an +jona than +la ura +vi bes +su b +coach ing +clear ly +uk raine +bra ve +commit ment +t all +mar t +ra p +mo di +sco tt +bro s +show er +ðŁı ¾ +âĺº ï¸ı +cou sin +appro ach +br e +com pos +hil ari +phil ly +g ad +quick ly +ri an +t m +vir tual +hou ses +k t +phoeni x +w ire +ff y +b unch +anc ing +tal e +snap chat +star ter +h t +k icking +ap art +th y +) ! +blo gger +it z +com fort +ang els +w ash +" : +ar gent +re quest +hon est +mi ghty +bo bby +k g +ro l +thou se +ex po +h c +tab les +mag ical +po sts +de m +n w +or lando +ab er +* ** +ðŁĺ ľ +environ mental +trans formation +mi le +w ic +hir ing +ma ine +bo ar +r ying +ti s +nit ure +twee ted +anton io +opin ion +fin ale +di y +f is +th in +trou ble +le go +fi les +qu art +sp a +curren cy +cli mate +fan art +rail way +sp ace +ban ds +dani el +mo tion +l eng +hol der +oc cu +mar ie +cathe dral +bu zz +bi es +nas car +bm w +bat tery +char lotte +doc tor +zz le +se ven +in san +d dy +st en +lab or +thr illed +se ren +docu mentary +wav es +cer tain +can did +allow ed +ninten do +star wars +ta p +home made +d les +ther ing +bre e +emp ty +pi ano +pos iti +coun try +por k +pu ts +per ry +m atic +spot light +ti st +or ities +we alth +c p +bar bar +commit ted +as sau +pro fit +e ight +hu l +fini shing +run ner +ss o +insp ec +char ged +christ op +lo sing +co al +ho o +ele v +de le +mo ham +don ation +c able +clin ic +j in +manag ed +ter ing +â ¬ +ur ban +depu ty +bb er +bur n +acade mic +o tt +sta ke +it er +sto wn +ack er +advent ures +ad ams +gre g +pro m +vo l +ac qu +con gre +pa int +citiz ens +c all +af ford +v c +as ks +the tic +independ ence +â Ľ +h itting +bl on +fu ture +â ı +in no +gen e +bo ards +di stance +se t +re mem +th al +pre vent +l ang +ob jec +su sp +mat t +in duc +bor o +pi one +re di +vir tu +prin ted +sco pe +shar k +suc ce +a stron +il legal +j ag +c ting +ine e +at o +rob in +nutr ition +b f +du tch +b n +fur niture +for gotten +at ar +ru p +hy per +bran ch +communic ation +degre es +on ia +un cle +promo te +or che +wi i +j s +but ton +ma jor +c bs +bri stol +premi um +ordin ary +e dit +m g +we ed +st even +: ' +gu s +te s +cap tured +dru gs +do w +wr ites +bi shop +whe els +ali zation +disco very +w r +rach el +ne il +hy dr +cu test +entreprene ur +kore an +ore gon +ul ty +perfec tly +suppor ted +histor ical +t wins +ell y +we l +de vil +in come +scienti sts +de leg +h en +on i +ic ed +gi o +cur ry +reve al +e g +buff alo +n ol +op era +camer on +haha haha +j ab +gradu ation +cra ig +r al +i f +organi zation +le ge +g ang +su d +edin burgh +l ack +fli es +g ate +thr ones +q b +the real +e leg +pp in +c les +jam ie +tn am +cryp to +ou l +p ages +a se +roo ts +stu pid +a did +boo t +prote in +s ap +si um +su s +end or +fun ction +don t +en na +ch y +squ e +wor ker +m tv +e a +k an +ðŁĴ ļ +mu s +professi on +t to +oper ations +al lo +c tor +inv ite +sc and +ou th +z im +lin ks +cli ents +sam sung +discu sses +n ell +ul tra +some where +ste wart +ine t +de z +b out +fac tor +ti an +tr ans +jere my +d b +ðŁĩ ¬ +or n +develop ing +spo l +coo per +ma u +rememb ering +tre k +famil y +sen iors +fo ster +att ended +w ing +trans form +ele mentary +hor iz +li sting +malay sia +it ch +warri or +philipp ines +russ ell +m end +initi ative +cre ep +to ps +br iti +a ur +shar p +adverti sing +ug ly +achi ev +materi als +bu g +dev ice +bon us +fac ility +col e +nh l +y as +plann ed +pol e +excell ence +tr ick +con fl +r p +achi eve +lo an +swa g +jess ica +ho we +p our +sc u +z oo +r ated +dre sses +re bel +mex ican +co ordin +me ss +atlan tic +t l +osc ar +wal ks +phar mac +investig ation +... # +cc i +eas ily +monday motivation +y ment +au ti +for ced +ar med +colle agues +pap ers +pro per +sha ke +bu c +le an +exhi bit +e vement +co tt +bi z +sp er +k ent +sw an +/ @ +girl friend +haw k +âĺ Ģï¸ı +mon o +ðŁĴ Ľ +stat ue +ðŁĺ ³ +ra s +te eth +preci ous +t ile +p am +swi ft +v ali +no se +dr unk +experi ences +come back +gen ius +wor se +sh ef +ra d +ed it +hon our +au spol +lar ry +h ire +gor don +achi evement +.... .... +su icide +alter native +su p +sur roun +sha ke +ke ith +pe pper +tur k +crimin al +be ck +su m +w alls +cn n +an tic +of fe +col li +win es +high light +hawa ii +emb ar +l fc +ðŁĩ ® +m v +> > +at mo +wor d +car l +shout out +bre wing +ì Ŀ +do f +s ic +hot test +col on +hh h +shu t +low ing +volu me +apart ment +agre ement +de stro +we e +religi ous +iow a +ro d +land ing +re present +ðŁĵ· : +la s +usu ally +h l +c ac +sal v +al ong +laugh ing +be ans +remin ds +pha se +some body +ma sk +ran ked +dest roy +sc i +âĢ¼ ï¸ı +gab ri +le o +ro a +fa iled +si l +refuge es +re vi +r ing +ber ries +coo kies +y y +conserv ation +sh ab +human s +de termin +a in +ni all +as su +mb a +fro m +extre me +vic es +commer ce +ght ful +or dered +suppor ts +re cap +v or +dro pping +correc t +pay ing +mean ing +n j +qui z +" # +busine ss +ðŁĩ® ðŁĩ +indi gen +du st +box es +bl ind +x xx +zz y +ðŁĩ¬ ðŁĩ +ss els +s ant +dd le +hilari ous +desig n +wonder ing +vehic les +k re +ju d +rece ption +par ker +Ã Ń +pri vi +hy dro +sof tball +pol lu +lo cked +ba h +e ar +scri pt +di vi +br ace +geor ge +the ast +bel o +j al +tion ary +dent al +roc ket +pur ch +sh ak +manufac turing +e z +it is +con cep +tb all +ch s +direc ted +pra yers +oo k +phil os +vari ety +che ss +ser ver +g and +bal ti +ðŁĵ ¸ +sel y +cru z +spectac ular +bur ning +re present +i z +t one +mer ce +h ell +bed room +estab li +bo l +com mon +ãĥ » +ab or +kit ty +hei ghts +re pair +willi am +qu ake +alab ama +popul ation +re v +re tt +i sts +n ite +le m +a ha +clevel and +r m +po ver +ob se +mon tre +man ia + ® +con ne +car ni +sh ah +f y +u a +sc or +strugg le +bo b +' ' +appro pri +deci de +ff ed +ca ster +s ort +hun gry +dra g +ا Ù +gr ounds +d w +sli ghtly +car din +dead line +bron ze +web in +bar ry +sil ence +e uro +op tion +ear n +ðŁĴ ĸ +howe ver +na ren +na ils +bath room +v ine +ph d +min ing +gar age +( ) +shou lder +defe at +di r +o v +liber ty +ple as +x on +com pre +a v +j in +ab les +sil ent +fam ili +vis its +di pl +ha bit +milli ons +regar ding +innov ative +sen ator +r ts +v on +k l +wh il +requi red +âĿ Ħ +lu v +presi dential +po cket +hun dre +sho wn +fro zen +to ward +fa st +confi dence +r ough +indivi dual +qu et +ðŁı ½ +dom e +fi fa +engine er +z en +re mix +ðŁĺ ĥ +pl ant +min or +robin son +as y +pul led +cer tain +potat o +( : +pre s +oc ca +w it +it em +si e +d ating +thom pson +own ed +an u +vi e +te dly +good night +ex cept +ðŁĮ Ł +ira q +ki e +ren ces +li p +simil ar +sau di +vi g +arth ur +pic ks +mil an +hon da +ma xi +o g +ste st +ar ch +analy tics +ba sti +pear l +ter ry +hor se +ast ro +ac ce +laun ching +inter national +s no +ta sty +den ver +ir l +pe te +tor n +advant age +var sity +" " +sol e +g c +lan g +demon str +ol ds +un ity +ne ts +insp ire +cre te +nash ville +nel son +e ter +wal k +hy un +m ack +tre as +see king +ra ge +bru sh +ab and +whil st +co con +h ong +shel ter +i p +possi bly +so o +it ed +â Ħ +rac es +war ming +qu in +tele vision +mat ches +ra pi +ment al +pal m +jenni fer +rol ls +indi ana +b ars +cat ching +resc u +candid ates +fa re +âł Ģ +se o +vie tnam +alph a +michel le +visi ble +re gre +wn ed +app le +li p +f fe +li z +york shire +ha il +se asons +be gan +m d +k c +la p +fascin ating +hel p +ur y +u ms +nu ts +se m +along side +bri dge +ori al +o ve +world cup +briti sh +comfor table +i ve +hot els +fair s +hor ri +so x +d ining +stre am +bar ri +ss y +w im +ter ms +v u +pe re +l ens +wal ked +r or +l ars +shi eld +dou bt +pro to +cro ssing +me ant +medi um +ad ding +e b +che ap +fun c +pap er +bran ds +ry an +feed back +col lins +un known +tro pical +sand wich +fal len +for mu +selec t +lo ads +answ ers +or i +mag a +d or +du o +ali e +dru m +ur i +de er +sou l +sh ut +âĺ º +sto len +don ated +bu zz +patri ots +ha l +na sty +nomin ated +mon te +ki a +th ri +ing u +te sts +pe tro +ðŁij ij +ho sts +ne st +to pic +pat ch +m my +hu gh +ab ilities +ma the +s miles +g b +ag enda +insi ghts +chi p +ph an +fail ure +dg ers +ha i +signific ant +sho ck +ru ral +gl am +figu res +pot us +o ta +mini stry +appe ars +fe ar +r h +americ an +h att +son y +fi res +e di +n ou +e qui +wh en +univers al +mad ness +i x +sculp ture +b ach +t to +swe den +et a +en to +develop ed +month ly +ma ps +ra h +le d +del ta +sa ints +is lam +ben ch +fif th +v ard +so cks +wel coming +j e +tur ner +v b +ad i +nor way +ad y +hurric ane +por sche +tra dition +ex am +newsp aper +lu ci +a ver +ide al +d na +madi son +ðŁ § +wit ness +ac ou +insi ght +si mon +robo t +sna ke +n bc +ac o +ro ss +sh ment +religi on +ch ann +in su +camp bell +inst alled +we ather +hor ses +ol i +rober t +k az +ðŁı Ģ +veter an +th read +quar ter +ea sier +cap ture +hi pho +law rence +roman tic +pas sion +cl ay +ox ford +th ai +stu dying +fi a +elec ted +most ly +c b +tu mb +âĢįâĻ Ĥ +x l +sh an +fa ster +ev ans +sli de +sh ri +see k +mi es +chemi stry +pump kin +tu m +, , +ro om +fi red +li ps +pres ence +af f +brew ery +arri ve +sw ag +photo graph +pen gu +chi ps +at tor +val ues +accur ate +con temporary +princi pal +cannab is +ari o +any where +gi a +democr ats +buil dings +li ved +ap s +neg ative +m are +bal lo +li on +diam on +loo k +re form +tom my +il la +tre ats +hundre ds +port land +wor thy +ex cep +ar ia +ido l +be er +cd n +y u +aw k +ðŁĩ ¨ +c ells +à ³ +ident ity +dra wn +de vil +f inger +th am +ðŁij Ĭ +ear ned +fin tech +dol ph +twee ting +evolu tion +ðŁĵ į +est im +m vp +n one +ðŁĩºðŁĩ ¸ +toyo ta +au x +mar in +b old +l bs +ste ak +mur phy +it able +lou is +sol ve +pi a +sk ir +ill ino +webin ar +ban ana +lo v +th on +vo ters +afford able +defe ated +lm fa +air lines +super b +any way +deb t +bo red +ver si +me tal +responsi ble +m k +s se +f ay +cau sed +f p +recomm end +pla za +spor ting +alli ance +au stri +n n +t ours +surpri sed +arti f +th under +sur ve +wor e +bri ef +necess ary +z ie +ash ley +dra ke +r t +kni fe +im mun +char ges +a the +bri de +rep ly +g av +broad cast +pu er +brace let +cap acity +harve st +id k +perfor man +d ding +il ers +par a +jam a +pro vince +ch in +id ers +har i +te aser +ch en +re stor +r at +fl at +col om +ðŁĴ ŀ +ðŁĩ¨ ðŁĩ +smoo th +r t +p itch +stay ing +isra eli +t cot +per spective +do ck +open er +lo vel +x o +class room +l ington +go al +kenne dy +sh am +sp aces +mitch ell +home coming +uk i +claim ed +recru it +ing o +mu fc +mon it +g roo +resi dent +per cent +per man +otta wa +int ment +an xi +stand ards +wor ship +sche me +f x +pot ter +bi an +athle tic +af gh +s se +sat ell +par ties +âĿ¤ âĿ¤ +infra structure +rela x +mo du +wor n +smo king +y ach +practic es +wc w +am b +dome stic +tay lor +k entu +provi ded +mo di +ve g +" ... +ob serv +ðŁĺ © +be ard +m our +an gry +ðŁĺ ± +startu ps +woo den +di ve +na il +anti que +ro ses +torn ado +m at +^ ^ +su spect +far m +de vices +me ga +tu l +scholar ship +ge e +disa ster +arri val +po in +mar c +kati e +bb ed +fal se +deser ves +ric hard +ju ana +fre y +tion ed +hy bri +r w +sar ah +ach i +c ure +o le +mor ris +ch ic +broad way +la bel +pa k +pover ty +gol f +e red +f u +er ies +be es +alo gue +st el +wire less +je wish +ti de +blo cked +life time +b har +sp lit +am ster +th i +jo shu +br unch +ha ps +s for +oo ps +ka poor +hi king +suppo sed +ro of +re as +tra in +ti ght +tru mp +bas ically +r r +ea red +see ds +entr ance +c p +wi e +son ic +vic tim +he re +e h +ear rings +sal mon +arc tic +an ne +dou gla +corru ption +hann ah +ha sn +vo ices +con ce +att a +fle et +clin ical +democr atic +ton y +st ood +le f +twit ch +a il +honest ly +incre ased +dro me +don na +accep ted +visit ors +ap ar +ad or +p ar +jer ry +ra i +brand on +ab u +!! !!!! +me me +in gh +glori ous +b hu +pu mp +j ol +li ke +fi sher +ma z +ag an +destin ation +play list +le tters +gen u +br ace +celebr ated +bann er +r he +dra gon +ðŁĺ ħ +sig nature +gre y +âľ Ķï¸ı +al ice +be red +ph er +ber n +ca th +ga thering +sc oring +influ ence +sm iling +de pt +lo cal +a x +ac u +reti rement +hon or +her self +chem ical +asse ss +y all +fre qu +appreci ation +ac a +cho ir +cu z +so il +c il +repor ting +u h +enterpri se +gr at +jaco b +ru m +fe e +j ak +sp in +bi kes +phi a +ste re +p is +bloo d +t att +ra ft +war ren +sh eri +back stage +mar sh +hash tag +ther ine +re in +game day +guar an +reci pes +min ds +stron ger +issu ed +bic y +n ak +ment ed +sc ary +u x +pre vious +tt le +th ats +ac tors +u ma +tin a +bun ny +promo tion +u ss +oli ver +montre al +what s +appreci ated +la kes +excu se +kno wing +pri zes +musc le +shad es +sco t +ing redi +electr onic +ju an +comb at +s ri +e h +turk ish +l om +stri kes +pri son +re e +po pe +vi d +ol dest +dol l +sw iss +certi fied +cli p +re turning +lat or +le igh +tt es +wat son +heal ing +el im +per haps +ha ss +k au +d der +mou se +new castle +indigen ous +wel comes +co le +tau ght +no ise +appe ar +jo e +can on +wedne sday +u tah +c tive +dri ven +i v +c ell +stri p +ac c +focu sed +ar rest +sto cks +wo o +â Ĺ +notic ed +shad o +di spla +ter ror +bor ne +secon d +que ens +wo ke +ja il +no tt +cam bridge +har t +se af +fa x +ac cept +âĺ ħ +goo ds +k at +t win +h s +thou sand +s ins +su ite +amp ton +ar n +rele v +ric har +hoo ps +n bc +class ic +p ab +soldi er +de plo +le ans +install ation +cla sh +le ban +ee e +ti re +belo ved +fu sion +travel ing +ne i +coo kie +glo be +phys ics +s q +co l +wol ves +d l +ex it +" - +foo tball +le af +ster ling +hi de +minne so +fresh man +natu re +indi e +supp lies +bri s +iri sh +ink tober +doo dle +ic op +mess ages +adul ts +recor ded +fix ed +ar do +offe red +under ground +dr one +p ine +ma inten +and re +ham mer +s x +r ound +hi ke +bra d +ro me +fu ll +on ey +ro ws +colum bia +archi ves +appro ved +bat ch +illino is +recogn ition +shou ldn +fo g +nca a +ke vin +human ity +al though +pow ers +p ou +s ar +pe st +alco hol +con sci +phil adel +en o +t m +ok la +cate gory +particip ate +accu sed +bri ef +po em +clu bs +consul t +ja b +big data +amster dam +ac ing +certi fic +n u +d at +impro ved +and y +campa ig +pale stin +p ace +mo bi +feel ings +wol f +bra in +pro pos +inter active +prin ce +inde x +c is +cha e +peace ful +co vering +ac o +cour ses +mon key +re place +b l +bloo dy +tal es +brigh ton +neighbor hood +g ates +spiritu al +af raid +bre ast +b ones +ðŁij ī +vide o +w au +tou ch +inju ries +car l +ri x +une x +âĢ ¢ +fre d +consi dered +thu si +an ch +on y +u sa +graph ics +ac re +ðŁĺ © +com memor +com mod +go ti +guar dian +star bucks +pre vention +haha haha +admini stration +portu gal +fac ulty +bet a +ul a +al bert +bre ath +er i +le tting +tr ic +ment ation +incredi bly +ten nes +v d +ðŁĻ Ī +ed die +br ick +gr ill +bt w +wat ches +resear chers +t ney +ni e +p as +a ster +vi br +poke mon +ch rome +go at +pitt s +il ly +festi ve +y d +can al +ðŁ Ĩ +fi es +car los +re que +partic i +tra ins +sam ple +temper ature +sym ph +pic king +in door +z ers +playo ffs +____ ____ +ap es +ly rics +islam ic +performan ces +d ick +spar k +se as +hom a +gr ound +disc i +employe e +com mu +alas ka +al an +fe ast +dg ing +ban king +manu el +slow ly +tru cks +mc car +oo o +sc rat +orche stra +indivi du +m x +bre ath +stair s +equ ality +bla ke +loc ations +cocon ut +balti more +aa a +l c +ðŁı Ĩ +har vey +resi st +immigr ation +adid as +fil i +re f +lg bt +mo s +pp i +ken ny +terr or +ban e +apol is +s g +social media +ka i +hon est +as sas +bol lywood +âĢįâĻ Ģï¸ı +ferr ari +hor n +cryp to +bo om +mainten ance +i di +s man +w l +ext ended +in sul +ve s +go sp +tr i +pi g +tar ge +cel er +st ati +sm h +ri dic +appe al +? ) +con clu +cos me +she ep +christop her +en thusi +po lish +me ts +oun ded +sustain ability +creati vity +con crete +ra i +ali en +ble ss +te es +clu b +ro t +bo s +ex ist +perfe ction +lu ck +rock y +expen sive +mean while +happy birthday +pre t +thr iller +ca ve +playo ff +som er +l u +le x +def ence +am writing +home less +pro phe +ch et +past or +ðŁ¤ £ +land er +ww w +Ģ ï¸ı +tic a +! # +o tic +rad ar +po sters +pow der +po li +ha un +tra p +bl in +assau lt +shor ts +re y +sh y +squ ir +rac ist +gar lic +fu r +remo te +sm ell +impre ssed +fing ers +âł Ģ +din o +le ment +s nu +promo ting +str ing +produc tive +b age +ma son +ra z +direc tly +j k +ev al +ðŁij Ĭ +doc tors +co w +ri der +st v +re move +w u +na than +ro d +n r += > +affe cted +inve st +mp tion +g inger +o d +agricul ture +s que +mu g +coun ting +ke e +mag nific +coo k +ani stan +roo t +plac ed +sym po +gh ana +un d +che er +thro wing +secre ts +f illing +opti mi +butter fly +bu bb +ðŁĺ ī +terri ble +d g +sil k +obse ssed +lo u +ai de +sal ute +mon u +philadel phia +scienti fic +i st +u ae +dess ert +bott les +can yon +ðŁĺ Ī +car ib +o ther +w ich +re source +guil ty +un d +le on +e ss +kan e +el e +tra iner +he im +an te +man age +roo kie +tre ated +po ses +rs vp +cau ses +aw ak +je well +le tt +on ics +tit les +cardi ff +g aga +bu mp +use ful +? ! +loo se +bb ing +: : +argent ina +de bu +cy cl +wh el +dis gu +j el +k ills +bio logy +ex ter +tra sh +bo dies +tr am +circu it +expe ct +la ds +w ells +sho t +ge e +naren dr +fa stest +b ent +b ills +mar shall +h ats +intro duce +citi zen +im possible +gi b +az z +net working +r ant +thin k +in dy +st ops +f theday +bri an +* * +amo di +dom e +coura ge +pac king +af fairs +g n +si zed +ent ary +pol and +swit zer +afgh anistan +w u +ten der +subscri be +mo sco +att end +republic an +hon ey +âĢ ĭ +si mul +we ster +foo die +or o +midd le +ab t +co pies +ma je +narendr amodi +ty pical +inspir ational +vit am +wis con +cu bs +tiv ity +h ali +e ars +k ay +d are +mari juana +cu rious +an ia +tom ato +re mind +ðŁĩ · +sc ared +cou p +po et +land ed +ri d +wra pped +mor ri +climb ing +e ws +fe eding +con tra +tho logy +gri d +ti vely +read er +la ser +di ving +di g +lat in +ti ed +shake spe +o ci +ad m +show ers +chu ck +mar cus +oo s +kne e +o live +ow l +dy lan +an no +g ym +deci sions +well ness +arri ves +sati s +chri s +thur s +ðŁ¤ £ +inter views +thank you +switzer land +over night +journ alist +ser ves +vol can +.... ... +plo t +nic ol +car rying +mag ne +tre asure +ex p +be ver +ðŁĺ ¢ +mar ty +mo le +don ations +recogni zed +b h +du s +sh ann +al do +success fully +ent e +ðŁĺĤðŁĺĤ ðŁĺĤðŁĺĤ +cab inet +cu is +tit led +d as +so l +strate gies +deli vering +ad ds +ani an +ne ther +ðŁĴ ĥ +con tain +su its +pa irs +to dd +rel la +ro pe +ci o +cro p +paint ings +su z +re jec +bu st +d h +fra ud +m h +contro l +je al +destroy ed +al lows +wo ol +minneso ta +om en +j u +sympo sium +d af +lim it +accoun ts +load ing +inter n +re solution +hol land +qu al +meet ings +gra ve +cam ping +v am +re nov +liber al +am ber +gre e +hu mb +fe ver +el ing +broo ks +à ² +be th +ad ed +al t +ro e +perform ed +jo sh +frank lin +nic ole +de ss +bb s +m g +net works +min im +al t +weap ons +gu y +jas on +g ha +harb our +at on +pra ise +kentu cky +bel fast +st icks +blo ss +ho pes +an thro +famili ar +wa it +ch ile +depre ssion +la x +je ts +le ice +recei ves +si er +an k +de x +inde ed +fle xi +fab ric +lam b +hel icop +am anda +âĢĶ âĢĶ +compe te +sn ack +techno logies +sy rian +mom s +mu ham +cho sen +an at +dev on +shar ks +re t +fundra iser +selfi es +st ations +communic ations +tennes see +tu tor +ro t +valu able +dynam ic +nur se +i ed +earth quake +deser ved +a ve +sar a +stre tch +dougla s +ne pal +à § +ob viously +d ame +ra pe +any body +k w +pat rol +hol ders +h anna +info graphic +ec o +be ating +stan ley +bo ats +ri bb +e z +wit ch +inv a +ac id +boar ding +- @ +gi l +da ve +care ers +opp os +l loy +in ter +do pe +re su +j agu +sh ade +in dy +on ist +rel ations +ag en +ab le +inci dent +me ter +shar ma +id r +pro ve +immedi ately +tro ops +am an +g low +gaz a +blo cks +person al +chron ic +all er +si d +sh r +whats app +lu cy +ar chae +ho u +journ alism +our selves +go t +the med +shap ed +we ak +cas ual +leng th +sla m +ab bey +e v +coun ter +est a +reci pi +cha pel +expan sion +sel f +suff ering +sp ice +n z +sp art +desp er +boo king +quart ers +y on +ðŁĴ Ĺ +p k +continu ed +- # +man hatt +tal ked +sh en +com bo +hybri d +je ans +liqu id +se al +re tweets +ac celer +collec tive +t as +: )) +profession als +ra w +o tt +su san +ir ing +okla homa +re ven +survi val +cre ator +tran sit +st ac +sur f +i k +ed iting +ch illing +bai ley +ste al +ra ble +pa rent +hun ger +sn app +collec t +philos oph +dedic ation +c f +c m +le ep +repe at +re ha +un fortun +a er +a ero +abstr act +mon itor +ag ents +bu l +sci ence +harb or +drag ons +floo ding +ac compli +d ash +juli a +the red +tues day +cy ber +b low +ta ined +le m +refe rence +pp o +ne goti +char le +con nor +au lt +access ories +commissi oner +rain y +re ar +advis ory +luc as +ma id +co al +k av +pol o +ðŁı ¾ +tran sport +mar gare +straw berry +bur ns +gre ens +ne v +partici pants +col in +belgi um +col our +in form +d ell +br on +cal y +kick off +strate gic +re union +hon ors +li b +egy p +âŃIJ ï¸ı +hy po +si zes +regi stered +bet es +relax ing +bloo m +inten se +valent ines +insan e +w wii +p x +tri o +bla de +wiscon sin +con e +plat in +ali ze +ra ven +incre asing +indi ans +il ian +bl u +rabb it +exten sion +je f +au di +fer ry +s ell +a day +us b +swe at +cham pag +metho d +mem ph +assi st +s by +ca pe +remo ved +mag n +v t +r ams +f bi +tack le +phe w +h on +motor cycle +su spec +eleph ant +sub ject +let te +da iry +whe at +awk ward +ac t +tro l +mit ted +zay n +sheri ff +ene my +con s +ke tt +bul ls +ev alu +bt c +satell ite +ho lo +por ter +dia betes +bet ter +rele asing +sur f +: - +se basti +collec ting +en cing +e thi +go ds +al ley +health y +m ills +sma sh +co pper +cr ack +read ers +sp ac +licen se +bas ket +bang la +en tic +om i +m ere +si vely +anim ation +lan es +dent ally +chill in +fi e +k aren +dep th +li pse +n g +ri p +mel o +sand y +ðŁijı ðŁijı +vin cent +nu t +hu g +who le +cre ates +? ??? +âĿ¤ï¸ı âĿ¤ï¸ı +bak ed +up grade +rober ts +har a +carib bean +auth entic +mb s +mosco w +attor ney +wi ki +ch lo +hu ll +cor k +" ! +sty lish +ðŁĵ¸ : +di ary +impro ving +ex pand +bri ght +pollu tion +k nights +person ality +chec ked +fac ilities +z el +bow ling +gu er +ðŁİ Ĥ +on going +un its +hoo k +be ck +confl ict +to dd +far ming +educ ational +k ak +cla y +stro ke +bel ly +explo re +mill enni +th m +loo p +sm s +consi st +cir ca +br yan +d ab +youn ger +soli dar +pp a +experi enced +b ella +bo ard +shef field +steph en +consu mer +sub mit +spon sor +t ang +ag gre +comb ined +trac king +sand ers +b az +survi ve +fer red +equ al +se p +re ed +str ong +priv acy +st ap +un g +ac ry +pa sta +pir ates +ag er +fair y +du p +introduc ed +wi p +let s +spr ay +ðŁĵ º +gre w +a sts +pitts burgh +new york +jo ey +lau ren +tra de +ch op +pi pe +cla ire +behavi or +v ap +cre ws +lap top +ðŁ¤ Ĺ +che ster +disci pl +d f +out doors +k s +go ver +super star +cas ino +far mer +; -) +re turned +ðŁı Ī +ma il +roa sted +co sta +v ill +pe z +gard ening +distribu tion +sh ining +inve stors +ra sp +dec ades +reali zed +bar n +p ti +st able +ut d +pan thers +m ens +b n +ca de +bu cket +yn n +when ever +wa ke +da is +ber nie +lo dge +ju lie +atmo sphere +ðŁĺĺ ðŁĺĺ +major ity +par ti +exc it +cu t +me h +musli ms +be gun +fli ghts +vene ss +ce me +po sing +so le +g ou +dark ness +pe ach +cel tic +auth ority +grand ma +ful ness +smi th +speci fic +gar cia +co ins +good ness +aldu b +recru iting +den nis +gar y +sle eve +weap on +pl z +disco ver +harri son +recruit ment +ja i +ch im +com pared +tom s +mo thers +am y +archi ve +t ask +ben jam +se g +law yer +al um +inve sting +mi e +che z +j p +a ke +fl am +wall paper +âĻ¥ ï¸ı +t ton +che st +favor ites +we igh +coo lest +r ating +relev ant +lo gan +ma ple +run ners +pri or +peop le +ma ur +terrori st +te sted +carni val +su spen +me asure +m v +cyber security +app ren +terror ism +o z +v ital +ni es +gon z +fun ded +twi st +assess ment +die sel +en for +colum n +ad dressing +ca sts +pay ment +x ton +fi er +, ' +la st +ne e +un less +clo se +sk ill +cuis ine +fun eral +ti les +a un +k ru +relation ships +ðŁĴ ¯ +ev ent +âĢįâĻĤ ï¸ı +kind ness +pro posed +acou stic +a es +defen der +dan ce +h tt +w at +vo y +ðŁ¤ ĺ +au s +cli ff +sear ching +beauti fully +in qu +at l +speci alist +ðŁIJ ¶ +da i +tra ils +class ics +inst ant +v ous +re venue +mar ch +kir k +fr inge +fire works +tri via +âĺ ħ +tr action +wal ter +mo to +l ily +att itude +cli mb +sc an +sav ings +c w +fa ith +cred its +ab led +gra ff +auto graph +he he +ran ch +ha d +ro gers +ðŁĮ ¹ +f in +re qu +fol k +ad ditional +lyn n +u ber +dol lars +lo gic +wor th +so m +the sis +p ound +bi c +st ur +cer am +spen cer +en tered +v amp +organi zed +âľ Ī +pp s +tr on +merce des +no ti +compet itive +do w +ous ness +vic tor +gr illed +na i +pu tin +ab ra +bl ame +alex and +anim al +dec ent +p ent +inter ior +:' ) +but ler +bal let +ðŁĴ Ķ +albu ms +down s +la d +si r +pla in +p ers +blon de +dis c +paki stan +se ment +ga a +w age +ch as +man i +co ps +terr it +lo l +lau ghter +ri vers +magnific ent +lam p +w b +new sle +char ts +ble ssing +p unch +lon gest +fl oral +cu tie +fare well +sto pping +mb b +bu d +chee se +de cla +si m +mc donald +de ter +you th +t ch +fre der +kin dle +fer n +at or +as leep +p ond +spr int +p ounds +la zy +gh e +fundra ising +dead ly +gran de +dou g +he y +lin da +consi dering +i um +gol den +vi k +auth ors +di ss +u ally +appropri ate +mor ning +y le +hon oring +foli o +be c +re bec +fin land +formu la +corn wall +sh ay +cau sing +bl end +sig nal +t ent +kash mir +nation als +har mony +sc out +acce ssi +he ight +medi eval +impro vement +ke es +prac tical +car d +de par +hu n +om ing +cal gary +ste l +bu bble +gur u +ma h +unex pe +n h +ed a +me at +i ge +si o +god dess +in ches +tun es +br itt +sti on +ra j +âĻ « +mer cy +ðŁĴ ĺ +sen ds +i est +pol ici +val e +reduc ed +as ap +vi jay +defen sive +celebr ations +ri ders +med itation +har mon +g ing + ¡ +program ming +in au +sud den +m h +replac ement +sk u +j ar +gra des +ta st +k itt +brand ing +k aw +boo t +f ought +p ays +g f +iz ation +ho p +k k +activi st +v end +coast al +cha os +ðŁĶ ´ +se me +bill board +li fting +cu mb +sc al +ðŁĸ ¤ +stru ck +l v +indie dev +beat en +jun gle +al right +destin y +m ing +k c +ch ances +om an +q atar +cra f +tra ined +pri x +char m +o tive +s mu +e c +and ers +hand ed +al ban +certain ly +arri ving +i ze +sa i +tr ack +pain ter +hu mble +appo intment +head line +manag ing +mo d +as pe +andre a +à ¤ +ethi op +un ited +exi st +bal i +k ad +n t +d red +re x +recogni ze +tam pa +be ers +ati a +he els +no te +transport ation +tur tle +re de +hipho p +sp icy +sp urs +⬠ĩ +cor p +ther n +to ast +hur ry +proper ties +ma ge +mar co +ele ments +bou ti +syn drome +ms g +develop er +gra ders +he im +re sil +off ices +del ay +di men +vin tag +barbar a +ðŁĺ ± +vene zu +cu lar +fac ed +bar n +ðŁĺ Ĩ +survi vor +wor m +confu sed +passion ate +Ø ± +identi fy +electr icity +sou ls +brad ley +repor tedly +lun ch +shel f +eli a +swee t +smoo th +emplo yment +am el +manhatt an +ste am +oun ts +ye p +li ving +un e +descri be +ca res +man ila +sha wn +ac ted +bas h +st even +re st +pet ition +div ine +wel sh +rac e +platin um +ðŁĮ ¸ +p b +extra ordinary +solidar ity +m all +on ion +schedu led +game of +fer gu +de ms +nor m +p k +tri als +polici es +publi shing +st ole +fron t +charac ter +van ia +ex ce +sti e +sc a +resi dential +sa iling +ðŁĶ¥ðŁĶ¥ ðŁĶ¥ +spons ors +th ick +champag ne +she pher +continu ing +ven ice +per th +na p +a ster +y ak +un limited +cho ices +ne o +hi v +repor ter +bru ssels +f old +dy s +se mi +la wn +it alia +wi fi +as k +em ed +fr ame +monit oring +ste ad +i da +gr in +is a +fli p +re stric +offen sive +atta ched +di sh +wh y +philli ps +gre et +p als +mix tape +v ou +fiel der +spar k +alber ta +g len +ca sh +s ri +u ri +ro dri +entreprene urs +climate change +p sy +d le +em ents +lin ked +nether lands +acci dentally +oppos ition +vel vet +ra ys +c w +om o +m f +lmfa o +newsle tter +: ) +toi let +liter ature +di sp +phili p +uni form +sudden ly +head er +cool er +-- - +prou d +bri g +nis san +scienti st +j ah +con centr +pac ks +appo inted +so ap +eng age +cho se +âĻ ¡ +se tup +jeal ous +har ry +g ation +tun nel +te mp +osc ars +dec ade +recomm ended +child ren +ab a +anxi ety +ve ments +sal on +pho too +organi z +mach ines +ab s +vil le +hy pe +ti ff +emer ging +av geek +[ # +contribu tion +bra dy +re sto +g mail +fit z +photo shoot +hel met +h t +eleg ant +ug anda +nur sing +or leans +pen n +na h +foo tage +em a +w o +w ad +concer ns +ve re +re mark +who ever +str ang +p t +qu it +sh ang +histor y +s ick +perman ent +ill ness +col d +visi on +he m +ar row +con vic +pin k +oc cup +bal d +ex hau +u of +am o +on t +ãĥ » +adop t +la id +smo ked +inter pre +ess enti +associ ated +b d +bb y +fi er +inst all +dipl om +con diti +c f +w ak +any a +gr aci +fi sher +s ss +ap r +il it +mus ician +symph ony +cor d +h ack +le gi +l v +bless ings +hum or +sc ra +e ti +min ster +trav elling +bu sh +jewell ery +li me +!! ! +pregn ant +pe e +lo b +cap ital +ip a +pen cil +la bor +duc ks +prou dly +wedd ing +dere k +m w +pe g +valent ine +an gu +re treat +pro spect +dang er +vul ner +up set +, # +sr k +x im +thur sday +n fl +kis ses +re ds +cr ack +re ward +c u +ko k +me te +aband oned +it t +me als +sp ell +stan bul +del ays +ru m +le op +gu m +no va +super man +ch ick +m is +dram atic +inno cent +r ounds +re c +auti sm +bangla desh +mor al +mo vie +sp oo +k la +âĥ £ +ou ting +mess i +ab road +loo kin +a im +q i +st ack +colla ge +à ¯ +hud son +sc an +ho e +ch au +oc cur +comm ander +ho les +ðŁİ Ħ +bi as +v on +stick er +ma k +responsi bility +colum bus +sa int +ed mon +rac ism +far ms +w en +gul f +may o +!!!! !!!! +corpor ation +ba chel +el a +inter nal +je ep +fol lows +di alogue +de rer +smart phone +he len +rich mond +equ ity +s land +b g +ne ar +av i +memph is +we ir +discu ssed +bad ge +p up +mi stake +phen omen +un ite +ðŁ Ľ +de pic +ri des +in augu +n at +sof twitter +comb ination +gosp el +âļ ¾ +ad mission +retro gaming +ðŁIJ ¾ +sch u +mb o +jun ction +al arm +à ¦ +gr ac +kh ali +k ul +m ale +cap tion +wi sh +te re +cor ps +ru bber +play station +er in +effici ent +l or +jo kes +in ary +nor man +lu is +inaugu ral +ch ed +âļ½ ï¸ı +di p +to e +str at +aa c +am u +pi er +co tt +comm and +tt en +sn oo +cu be +clo ses +class ical +s word +expre ssion +reach ing +n app +co st +affe ct +ric o +gi f +brea the +tri be +or tho +h ay +l g +fri es +n m +hi ding +richar ds +en de +mic ro +capit ol +cop y +ro m +regi me +mary land +tax i +di al +embar ra +un believ +ch t +v s +elim in +o dd +pen ny +sound track +l ings +trans ition +rema ining +a is +mali k +? !? +rand om +def end +ul tra +tru m +danc er +st ol +dri ve +a ver +ro ast +defin ition +se an +excit ement +partic ul +su rely +sh av +ber y +di shes +com m +is ol +i am +ob li +gho st +hugh es +chi efs +b as +conserv ative +speci al +fe min +sh ri +n ancy +inte l +tu ne +ðŁĩ ª +jo el +gg le +mo to +ðŁĺ Ķ +bu ck +d ag +antic ip +mont ana +gu id +fro g +ec raft +op e +dri ves +nu mer +x y +color ful +wednesday wisdom +illu min +bey on +inau gur +deep ly +pre fer +for tune +coo ked +ti ble +âĺ ķ +swe ater +it ter +tt y +u i +gi e +com plic +~ ~ +tax es +cu ps +di verse +sam anth +âłĢ âłĢ +ba king +sy mp +wa i +be half +mer cur +travel s +ðŁİī ðŁİ +or ia +eng aged +jump ing +reti red +n aked +p uni +speed way +sci ences +rehear sal +on ym +dy ou +pl ates +r ati +kri sh +jaz z +car ol +ra f +pen alty +tim eline +ru by +engine ers +ra f +bel le +do se +che on +esc ap +me g +ran k +or d +me gan +mer ch +ec lipse +âĺº ï¸ı +ple dge +kir k +per si +leice ster +sa k +w k +saf ely +yy y +je t +promis ed +j c +en ne +no ah +re no +re a +ðŁĺĤðŁĺĤ ðŁĺĤðŁĺĤ +tra il +ðŁij Ģ +f d +soo o +ri min +w k +ภ² +i al +x ox +bis cu +d ale +fan dom +particip ating +fla g +privi lege +pe ach +mach ine +bo ston +gro ss +o g +mir acle +adop tion +u ss +mon sters +be ij +clar ke +pu shing +pra ying +ar o +d n +ell is +apol lo +od ds +refuge e +to w +b p +ðŁĩ¬ðŁĩ § +h end +app eared +memb ership +pe an +du m +viol ent +v y +potat oes +aw w +greet ings +t ts +ac on +sh ane +photograph ed +cra b +temper atures +cu ba +c fc +wel com +he l +in nings +m k +co de +kno ck +gra ss +swe dish +p ta +ick y +v at +lin ing +s q +sa p +ar c +announ cing +sk ins +cit yof +br ing +co x +gam er +it arian +i da +h d +ros se +sad ly +ge o +âļ ¡ï¸ı +tag s +fa ther +chan ge +l ance +whis key +adel aide +te c +stick ers +marke t +class y +bad ass +flo rence +lin er +fro st +k ate +ac on +scand al +es sex +ðŁĺ ı +vi vi +dr ill +blo ggers +recomm end +d ha +ac res +ro ma +bu y +gro cer +er ia +ma har +ff er +patter ns +ver i +com pu +st ev +ang a +ment or +do o +it ali +cdn poli +on ly +conduc t +elec tro +de f +wh ale +prepar ation +bicy cle +vi ral +turn out +bra ss +qu ad +hospit ality +pack aging +den cy +ceme tery +abo ard +dre aming +pic ture +t all +inv ent +ad mi +o e +tem ps +qu an +fun dam +pro mp +resi dence +mu d +sour i +âĦ ¢ +graff iti +gi f +d nd +com p +s war +pe eps +pale stine +devil s +san g +assi stance +bi ke +missi ssi +inter viewed +ne phew +dru ms +v and +gentle men +n sw +inst a +leban on +ee ee +oli via +ver y +rou gh +industri es +m ation +ðŁĺ Ĵ +bar rel +n ay +po ps +moder n +ill y +are st +on ents +protec ting +v ans +e o +vi kings +restaur ants +re ck +jac kie +andre w +w illing +he ath +citiz en +disc rimin +๠Ī +stu art +m ys +hi p +tran sp +" ? +te x +su shi +ke d +cro ssed +dist ur +pe dia +f ate +some how +mo th +proce ssing +is s +r in +u ts +yy c +ver t +lg bt +re id +on to +arab ia +habit at += = +stre ak +simp son +addic tion +wim ble +deli vers +challeng ing +ðŁİ ¶ +fran ch +e du +s me +ai ds +hur st +th am +tari an +remem bered +palestin ian +fe es +tru m +sket ch +ur u +fit ting +jes se +ðŁĶ¥ ðŁĶ¥ +---- ---- +ba ch +ici a +colo red +da h +associ ate +int el +s eller +p u +stu ffed +ac s +b s +sh in +cooper ation +certific ate +ab u +ingredi ents +re v +in ge +el der +christi an +bun dle +th ic +dir t +beij ing +comm it +ted dy +ed u +to day +s field +w yn +confir ms +lo o +j v +ene ss +al pha +vir us +ari um +gr ind +bri dges +introduc tion +pol ls +bac ter +z ach +termin al +ra iders +fla vor +zom bie +vo d +sp reading +gameof thrones +effici ency +lat ely +ale m +twee t +cri mes +cl er +de y +dg ed +hy un +pay ments +cir cus +ðŁĺŃ ðŁĺŃ +mis souri +lu b +episo des +c age +po s +mat ching +tumb lr +lin ed +ge st +am bi +nar r +ing ton +regu l +blo wn +is le +co co +on don +joshu a +tour ing +sm a +sau sage +best friend +bo eing +desi re +sav age +ra pper +de vo +te ar +take over +cow boys +po ker +par ag +pp e +h int +we ars +se th +ro les +l anc +man ga +form at +fl yer +c ay +mo or +ba ke +spla sh +v ad +ker ala +proce eds +sil ly +reflec tion +di str +wi d +su it +ci vic +yan kees +by n +migr ation +di stin +or ch +fe mini +quali fying +tu ri +o be +hun dred +cra p +wan g +mathe mat +bu re +expo sure +fergu son +seme ster +re serv +pl ym +a hu +fac ial +wa x +wor ried +ca b +vi o +as a +co d +to pics +p cs +hal o +rescu ed +horiz on +ar k +âļ ª +hol ly +el f +ul ti +pu p +quali fied +attend ance +ati vely +destro y +y c +for th +photoo ftheday +c ents +ic eland +meas ures +de sk +port folio +artic les +direc tors +dat ab +e w +creep y +oun ding +hon oured +mi st +j it +men tioned +port able +iti c +d ann +friday feeling +am id +ti ger +scri p +helicop ter +hard ware +expl or +work place +austri a +beat les +ber nar +spi der +disc o +cul t +lim its +shor tly +fin al +nin ja +lu ke +le bron +wal mart +o il +van illa +shi re +ye g +ak y +c s +bl er +collec ted +t g +rol led +speci als +b ff +pier re +sh im +vi er +flash back +restor ation +individu als +pro d +fre aking +tu rer +o a +re fre +mor oc +gre et +re yn +care ful +our ing +u sh +is d +g ill +vie w +thunder storm +b led +pic nic +guar di +pi g +ar k +syl vania +bann ed +u cl +vi jay +ori um +av engers +believ es +eu r +monu ment +concer ned +la bs +ber g +a ap +vi sh +sing les +can cel +z el +ar ab +ru th +too th +ar ta +sh af +chair s +r ack +dise ases +crow d +cl y +fle x +christ ma +artif icial +tom at +fin e +dra ws +advoc ate +fran ce +Ù Ĭ +ðŁĺ ³ +heav y +s our +compre hen +no ble +aa p +hin du +cor al +g ars +ow en +n l +st all +yel low +mar ina +in ver +suppor t +tou gh +promis es +pi e +master piece +sco re +for ce +mor tg +crypto currency +o x +r ors +rock in +pro vin +ho g +no stal +oak land +pat rick +inclu sion +tra ffic +ah med +a ha +lux ury +con secu +de mon +âĸ º +b lowing +st ag +: " +encoura ge +ben e +sku ll +do dge +bu ster +kin son +wit ne +er ror +lo west +fel low +à ° +sh re +bl ur +vir gin +compos er +sli p +mor nings +ga ins +tab le +gra in +ari st +braz ilian +w we +tu es +ribb on +an ag +di st +sac rif +em brace +entreprene ur +af fili +de o +t ali +touri st +fat al +ì Ĭ +autom atic +ðŁĩ µ +we ak +wel fare +confir m +benjam in +fi ghts +alleg ed +me ad +strugg ling +pro secu +che f +à ¨ +propos al +er n +ðŁĺ Ħ +dy k +on gs +hon g +m ack +mel on +on ent +ru sh +d ap +tol er +pro pag +c ze +trans lation +wal let +cott age +sa il +constitu tion +ðŁĴ Ģ +mun ici +fav or +storm hour +i h +ðŁĺ Į +approach ing +pin ned +j ed +niger ian +n ach +sh at +particul arly +mc don +camer as +anni e +admini str +he at +electr ical +char ming +gib son +bouti que +ex posed +ac tor +pil low +beach es +genu ine +margare t +ben nett +lou isi +pos itions +el y +shin y +ten tion +architec t +ren tal +ac qui +goo gle +sub way +mom ent +ðŁļ ¨ +ri m +metho ds +cy cli +nor folk +Ù Ī +over whel +ra pid +we ar +happy birthday +progre ssive +ðŁĴ ¥ +co gn +pap a +f ool +philosoph y +pol ar +jim my +wi g +ðŁĴ ĭ +oper ating +reduc tion +ph i +fla gs +to the +o di +a res +k oo +k ang +ar kansas +ash ton +wimble don +sci fi +attrac tive +mississi ppi +logi sts +ral ph +la bel +gradu ates +ma ha +home town +âľĮ ï¸ı +foun ded +on the +li z +trans l +mini mum +pre sti +ta m +gener ations +re bel +journ alists +par am +mc m +acry lic +death s +tes la +w t +bry ant +jer us +i stanbul +muham mad +ri ley +k ris +work shops +is o +coun ts +stre t +prote cted +trin ity +man ual +r hin +r il +pleas ant +le mon +ner d +har der +dar ren +bur y +ra h +bas is +mi gu +occa sion +li sts +âĿ¤ï¸ıâĿ¤ï¸ı âĿ¤ï¸ı +e b +de cre +hamp ton +ìĿ ´ +tra vis +trans form +puer to +nh l +av oc +tri ps +unexpe cted +ve t +di dyou +bar ber +st ages +m son +re presented +for t +l al +pp le +nic ely +ignor e +qu il +qu inn +h k +carri er +remin ded +am ong +pass enger +el len +gue z +sc ape +mu ral +youn gest +ma sh +d ill +rout ine +stain less +jack son +gand hi +th al +on ers +edit orial +convers ations +sd ale +autom ation +i ke +า ภ+ðŁĩ ª +hau l +la ying +men tions +am en +abor tion +i bi +coun ties +ca therine +man ds +jam e +roll er +au t +n am +o logical +cep tion +ran king +tox ic +sn acks +victor ian +bang kok +psycho logy +re g +ang ela +respon d +sty le +sophi e +dak ota +achiev ed +mar ked +imper ial +in as +glo ves +sli m +confi dent +att acked +gg er +lon ely +valentine sday +re b +craft beer +orig in +zim bab +ce iling +te ens +other wise +w b +f ers +day sof +advis or +y ah +âĻ ª +en der +republic ans +av a +skir t +pi pel +chi e +jan e +ja x +ðŁĺ ĭ +âľ Ĭ +j ays +bre tt +bal o +cru cial +d har +as is +de au +lloy d +chat ting +âĿĦ ï¸ı +rel ay +remark able +n s +we t +bris bane +ðŁĶ ´ +tion ally +f k +la yer +house hold +consecu tive +es is +pend ant +st ir +crit ic +su gar +photo shop +pa res +arti stic +do dgers +c un +cra fted +am end +bo at +âŃIJ ï¸ı +egyp tian +sa w +tra ge +small er +ox y +pa ired +nex t +i res +tac o +o y +u c +st i +a erial +: // +dr o +dot com +gg ins +r pg +ay e +le an +stri ker +lo bby +prote sts +pri ority +congre ss +am ate +inv it +r ington +mom my +th us +allow ing +pione er +enfor cement +g ori +tal k +dra g +du mb +bul let +san ge +er y +tar gets +ðŁĩ ¦ +he ather +consi der +seaf ood +ve st +ris ks +% . +p g +sac red +he ating +kick ed +tto t +. - +chan di +co ven +po ol +pul se +i a +ro ster +shakespe are +es a +car go +pean ut +tro op +ac tion +tab let +home work +cast le +stru ction +mus icians +free zing +bu tt +justin bieber +j j +bah rain +an them +au dit +didyou know +na vig +guid ance +âĸ ¶ +tur f +n un +fic ations +ye men +char ging +x c +bron cos +su bur +p ale +bor ing +among st +for the +em per +om fg +p j +expe cting +ðŁĴ « +st l +ad min +expect ations +sw an +shoo t +oooo o +min ent +ãĢ IJ +wall ace +stan g +satur day +adop ted +dou bles +hom ie +ome z +d han +vent ure +surroun ding +fi le +mob ility +de es +w ski +broo ke +emb ro +re members +kar a +test im +bo tan +m tv +sacrif ice +jerus alem +d l + ´ +proper ly +ili on +as i +leg it +co pe +m cla +recy cling +lar ger +ðŁĴ ĵ +pat ric +gener ous +ja red +p f +mol ly +thom as +ju dges +h b +sor ts +bl vd +o ven +enter ing +plan es +be et +integr ation +boo ked +fre ed +ver n +ash es +to pped +de pot +welcom ed +ren a +m ick +d and +see ks +gam er +ran kings +ren e +mu t +whis ky +fire fighters +gu es +ga ther +tour ney +de men +y ang +new ton +autom otive +back yard +deta iled +mi st +to bac +fi ber +un usual +grat itude +sp are +ne ys +: * +per i +flo ating +fin alist +don ating +dre ss +bro ad +be the +econom ics +tai wan +ed wards +plu g +pra iri +val en +bab a +f ad +an as +har per +dis order +app lied +p att +bi kin +li ver +cu ri +carol ine +ann er +juli an +wal king +mal col +screen shot +co ding +skin care +activi sts +myster ious +ex act +blo cking +mercur y +bat ter +du mp +âľ Į +en se +li sh +ridic ulous +prote sters +ðŁĻ Ī +lu st +swe at +as s +ali ke +co dy +re ments +win ds +as pir +vi enna +pra y +.. .@ +bo i +cand le +assi sts +te e +der son +p ony +f ence +con spir +âĺħ âĺħ +oo th +e pic +ba rely +a unt +b am +diamon ds +end less +scre ens +can cer +gr o +p st +pro spec +mo sque +help ful +ou ri +bro ther +gu jar +cri sti +ine z +to wers +ad dresses +gra y +bur ton +re tweeted +ðŁ¤ Ķ +n ity +du ck +super vis +jo an +kin der +sanc tu +pi ed +âı ° +ł ï¸ı +m ati +reven ge +ce ster +eli fe +desig ners +back ed +bo li +wei ght +cou ch +su res +s its +shri mp +la gos +auth orities +os ity +hol ly +compu ting +fac tors +ab e +pan els +ram ad +sent ence +missi on +hol m +r b +d ads +shang hai +mon ey +she ets +sk ate +thre w +cup cakes +infin ite +l is +practic ing +ess ay +ka i +as ci +mo b +u gh +hol mes +re gg +ik h +mo ck +collec tions +pe p +o va +sal t +nan dez +co y +thre ats +tex ts +cin nam +pregn ancy +pen ding +stam p +flow er +g is +agre ed +pay ne +ro ver +ph ra +sof t +f fin +fa thers +pass engers +aw ays +al a +h es +li van +in s +samu el +ingu i +h of +j j +chen nai +cat al +om ic +he ath +ni ece +pump ed +integr ated +are l +no m +produc tivity +wan ting +vis a +di ana +tw il +it v +cam ps +ro wing +d ley +black and +gu ards +b ells +re verse +vi be +ric ky +mo ss +ny t +âĺ Ģï¸ı +el le +tro y +cu dd +ev an +women s +fo to +mi stakes +wick ed +mi l +c led +me mes +co smo +schol ar +ren o +ðŁĺ Ģ +v ents +# âĢ¦ +terrori sts +ca sey +cardin als +ðŁĺĬ ðŁĺĬ +venezu ela +bol a +liter acy +t w +en o +con tains +au stin +fin anci +ev an +har vard +origin ally +chev ro +her ald +nott ingham +manag ers +âŀ ¡ +accep ting +wal sh +tutor ial +entrepreneur ship +yach t +requi rements +glen n +pe de +unfortun ately +ach ing +dais y +gi an +night mare +âĿ Ĺ +r ina +b art +ema ils +oppo site +who m +sa ke +pu zzle +da shi +par ty +blan ket +bus es +lo re +beau ty +reas on +pun jab +winds or +func tional +exi sting +hel lo +gli mp +con vin +la k +scre aming +rebec ca +bli ss +north west +infin ity +cosme tics +pul ling +coffe e +pl ing +op ho +colom bia +interior design +( + +emo tions +sa c +sun glasses +sav es +d f +six th +al y +ðŁĺ » +de en +dev ast +polit icians +lac rosse +g u +pe i +jav a +comb ine +coal ition +er ts +survi v +ch ad +stri an +n n +de vi +coun c +concer n +contro ller +bre ast +j ury +tu m +introduc es +la di +mobi le +al z +ste ady +nur ses +h acking +on line +oce an +ðŁİ Ħ +a am +ju ven +ic c +louisi ana +ar te +street art +is on +wn s +fr m +p anda +no ir +main tain +del ay +symp toms +thor n +ge ome +ter n +carri ed +p ru +pan or +as sy +per u +clou d +sp ra +pe di +e ste +tag ged +ðŁĺ Ŀ +shado ws +naz i +ا٠Ħ +cor ri +âĻ¥ âĻ¥ +j ad +ðŁĩ « +form al +spo ken +ðŁĮ ŀ +enjo y +lo pez +out look +in ho +w ander +Ù ħ +ma ya +pe e +d ine +ãĢ ij +brief ing +suppor ter +ar ily +ght ers +natur ally +doctor who +j en +v ar +new year +re se +si mm +re x +con sequ +tomat oes +bur st +bra vo +bur gers +cr acking +nor theast +bi om +mush room +mar que +dou ble +ni er +v ag +tw enty +key board +win ni +jama ica +par ish +: - +mental health +ali zing +ren der +wa king +ðŁİ Ĥ +g ly +na than +wa shing +mel issa +jun g +loy al +chil i +song writer +guit arist +bo wie +neighb ors +onym ous +as set +ta i +head quarters +ðŁĮ Ī +i hear +ci gare +sur g +) " +re pl +dar ling +ðŁĻ Ħ +z ak +sa re +ãħ ĭ +mic key +ware house +mass age +ine es +did nt +i w +hur ts +eng aging +mag ic +women in +k itten +mor s +c art +tit ans +colle ague +compe ting +er an +k hal +mar ble +dem and +del ight +et ary +bli zz +lou ise +m ls +fini shes +experim ent +conduc ted +electr onics +itt ers +car ing +wh ats +sym bol +jun g +e cu +pi x +con text +char ger +ðŁĺ ĩ +re ig +fra g +ë ĭ +ch ad +tru e +ker ry +def ending +a int +au ton +check out +bar nes +less ly +d t +m me +clou dy +second ary +are z +_ : +app a +const ant +" ) +ve ts +jo b +i ent +ðŁĺŃðŁĺŃ ðŁĺŃ +m j +fren ch +di ver +davi es +hh hh +e book +๠ī +mar iti +bree ze +susp ended +mat o +vi et +ra hu +se i +bol t +en ary +le is +kar l +fr amed +expla ining +ab c +de aling +nat o +ja ke +exp and +leon ard +establi shed +du b +ar men +el led +voc al +nichol as +ori ent +k yo +illustr ated +ah h +danc ers +milli on +ge ta +po pp +as u +mur dered +gi ble +sto ked +gri ffin +maxi mum +adri an +en counter +ther o +david son +ðŁį » +holi day +ev o +asse ts +car son +memor able +âļ ½ +ob am +represent ative +cb d +tr icks +vo gue +vo ice +mm mm +sebasti an +cli f +ath y +par alle +ðŁ¤ · +pa k +ev acu +e ats +ا Ø +tou ched +organ ised +spir its +can ad +gui ded +frame work +ðŁĮ Ł +pe d +natur al +ag ar +replac ed +anch or +ti t +sha h +organ is +super ior +r n +ch ro +eric a +st ill +cor on +chu ck +loc ks +or gan +ro sen +sc am +ben ed +/ # +ke en +tre vor +vamp ire +sor ted +! ' +af ford +in tro +gr ace +ðŁĺ ľ +sau r +kick starter +influ en +v u +y up +po c +ðŁİ ¥ +a ar +s ang +tre k +et sy +tb h +scre am +chevro let +pix el +shepher d +an or +gabri el +tw ood +sd cc +me ters +develop ers +clo sure +v w +twit ch +ì Ĺ +se oul +pr ice +ho g +n ish +hill ary +scrat ch +in cen +wag on +dis ability +pan ther +ch ats +g d +wit z +sus sex +l ate +den mark +ger ald +cancel led +net te +i x +nav al +bap tist +te t +y ad +ma th +ho y +r andy +po int +intel lec +fru its +w ool +gu in +pr on +the ft +con dem +mar ry +n ola +architec ts +cin cin +roc kets +gentle man +ex plan +t ate +do e +ra ises +wild life +w l +insi der +blan c +w p +for sale +ny c +po well +unbeliev able +pen s +goo dies +mu stang +p ens +st ays +squ ash +xox o +near by +ever ton +co co +le agu +k han +stu d +south west +con struc +s worth +cro atia +le a +su ms +aim s +e an +van ess +iti ous +pa thy +arc ade +b end +sugge sts +sac ram +roy als +ri er +em ir +in cl +an k +clar k +ri ght +vac c +ठ¾ +tan e +li b +u sc +sal es +hu h +s ally +ver a +p ga +gro ws +dru m +tre e +eth ics +sug gest +is ab +se aled +pre viously +anim ated +ab du +ri ses +glo b +pre dat +scar f +del ic +om ar +ll i +sx sw +py thon +ne bra +fun k +reflec t +pav ilion +tic ally +ch asing +bak ery +inva sion +ko h +believ ed +co hen +con qu +cra fts +nat i +cle ver +govern ance +sam ples +fa ils +â Ķ +ti mo +r itu +stri king +inclu sive +sho cking +can t +requi res +dra wings +à¸ Ń +purch ased +du m +z ach +war ner +con sole +man sion +foun tain +circu m +e sh +is land +mil k +pro fits +hali fax +ri val +âľĪ ï¸ı +jen ny +sand ra +ny e +k elly +y al +qu ad +no s +inste in +fin alists +mid fielder +cu e +excep tional +a an +sa pp +gett in +sa a +f ati +sl ice +vol k +s wal +la sting +sum mary +it as +sm o +s z +âĺ Ĩ +ip l +fl ames +ene ws +ha v +hoo die +pitch er +win dy +re vol +centr al +ton ite +ðŁİī ðŁİī +sol ved +mil wau +organiz ations +wee ts +re fin +s th +ãĥ ¼ +el in +ton a +cinnam on +ðŁİ ¨ +ðŁİ ģ +ron aldo +pen insu +ome ga +el ds +desig ning +e igh +blu et +ben z +nu g +ash a +robo ts +su dan +choo sing +en do +ser ge +clo sely +hand y +fing er +be ing +ar te +survi ved +fl ame +mile stone +gu t +d war +fu tures +é e +el o +fri dge +eli c +ou ch +u b +p v +tit an +col lar +st ation +nev ada +aur ora +r d +dun can +âģ ł +bri en +mar sh +Ð ¾ +to tal +ch ry +s ers +su ffe +ra chel +colle ge +to days +cour ts +ch it +re united +gym na +gen esis +be side +re presentation +ch ant +collec tor +ra k +ath ens +ni gh +mun ich +langu ages +fl u +particip ation +__ _ +c v +spec trum +so da +co ver +refe ren +ab bo +ap a +public ation +ed m +mon ica +ar my +ðŁļ Ģ +div or +dr y +stre ams +robo tics +ci der +bull ying +appro val +sto ke +plat forms +sier ra +ex tin +i b +ha yes +succe ed +suff er +at ically +da i +lyn ch +h ound +del ines +ack now +d ated +exclu sively +he res +fac ilit +dam aged +char ter +la kers +fal con +unve iled +wel ove +e ase +pati ence +l one +gent le +gene tic +produc ing +g our +shann on +bil ities +zimbab we +p int +dau ghters +liter ary +bel le +cl am +surroun ded +k any +ne il +pir ate +rang er +hb d +nat alie +bel ong +olym pi +emb assy +sc ol +en er +ak in +lo ren +b h +: / +di va +den im +hi pp +ðŁĩµ ðŁĩ +arn old +? ' +we ren +em power +dis abled +man or +rasp berry +b af +aw ful +dru mmer +kar dashi +n ash +machine learning +ch u +rebel s +tim ing +mon roe +ton gue +ran ge +pup ils +re ss +amaz on +b z +har ley +pal mer +ballo on +s ings +ic ec +j b +c ers +g ps +whi st +ri se +l t +oo oo +c attle +shoo ter +vod ka +uc l +mt g +le sli +jon as +di spo +at ric +ste in +vintag e +fir ms +flo yd +cow boy +soo oo +is aac +war craft +disney land +beauti ful +be am +franch ise +bu n +k ag +an on +tur bo +swee p +made in +kar achi +dete ctive +penn sylvania +contro versi +vitam in +a side +chron ic +descri bes +remo val +ha h +ap er +ten ed +u to +bad ly +mir ac +f ry +ye a +in jec +ther mal +comp act +th or +te ed +ur gent +l ite +g illi +sop hom +ic o +che m +p m +for k +fre ak +ch ak +recipi ent +i y +ni k +model ing +c ans +ðŁı Ģ +del ux +se am +surviv ors +rad ical +investig ating +reli able +f m +tur t +ligh thouse +to ol +go wn +) ) +bo ts +auto graph +a id +bu ffe +h mm +horri ble +ssi onal +ann i +๠Ģ +k its +sch i +eter nal +hu ss +sens itive +r u +tast es +chec ks +im o +por tion +sk ate +e den +half time +fri ed +ri hanna +ti se +fl ick +ca in +s gt +âľ Ķ +sh au +sta ined +ra ffle +dro ve +sal man +princi ples +sh o +ar u +je ss +gu ine +gar bage +my an +jel ly +dis ru +z ia +q ld +ent ries +la v +fle w +ad mit +objec ts +comp are +ny times +cann es +p n +suff ol +ro c +d ana +e gg +hi st +coun sel +' ! +phy si +imag ination +ad just +explo sion +plym outh +hor ror +elli ott +bour ne +de x +bre ed +au dio +lob ster +disappo inted +nation wide +( ( +incre ases +austr ali +ce dar +star ing +rac ial +e is +g mt +visi ons +stay ed +discu ssions +de an +cur tis +mai den +stel lar +happ iest +h wy +pre season +car av +mon days +hospit als +glimp se +schol ars +ja i +ter race +ann a +goo se +gra ded +lot us +hun g +grocer y +stam ps +emper or +sc oop +in ser +c as +exist ence +he al +fal cons +mar vel +reduc ing +terri fic +magne tic +perfor ms +bar re +p us +tre ating +ic on +w h +decla red +tra uma +do d +come dian +nik on +bu gs +as m +mont gom +ibi za +comprehen sive +ha s +san ti +fellow ship +da sh +p sal +louis ville +sp y +fau lt +d the +fi led +vi sta +de sc +fe ars +you tu +sp s +es p +ri g +cri me +ber ger +wonder land +k ent +in formed +stev ens +my th +ast on +ir i +visit or +at ri +produc ers +al la +person ally +separ ate +agen cies +af ri +il an +spo ke +n ina +squ ad +di ves +de pend +li v +fier ce +enter taining +cha in +sc at +bor ders +pal ette +sp ro +os is +der by +tobac co +zi o +willi e +ju vent +zoo m +hol y +enti rely +af e +mart inez +be ds +pe a +bull dogs +ðŁĩª ðŁĩ +ib m +ne on +ethiop ia +team mates +plan ting +tw er +any time +for bes +ó n +run way +ner vous +ro ger +p ile +ch anc +apo caly +u w +o i +dr ought +territ ory +br ick +cre atures +go in +w aff +gre n +sou theast +je an +am bul +ed ited +stra p +c v +aar on +ãĥ» ãĥ» +t su +descri ption +kin dly +clu tch +im mer +en or +women sday +or ange +ra g +ob vious +hy der +chann els +man go +me yer +ra ining +ge tty +pil gri +coordin ator +up load +ninten do +don uts +san chez +app arel +j r +zz i +, @ +jeff erson +accessi ble +great ly +e id +initi al +budd ha +par is +ma scot +â¬ĩ ï¸ı +sch war +si ri +sp inning +mortg age +e cho +end ange +ge dly +chlo e +enh ance +kar nat +k ry +explo res +ðŁĴ ģ +af fair +ic als +all a +dar t +dolph ins +diffe rences +squir rel +au gh +dr ones +ell en +re store +pa w +un for +pi ke +hil ton +colla b +consu mers +co inci +out comes +pp p +a q +coup on +li est +si ms +k ho +av es +spo on +pu dding +cor byn +hat ers +ex ams +sla ve +. ! +p sa +app les +tam il +se d +co ke +zz o +lo sange +car bon +cla ir +... ) +k hu +cra ig +explor ation +sanctu ary +su e +al way +demen tia +won ders +super hero +pakistan i +brown s +bluet ooth +lo cker +mar c +ev entu +delux e +rodri guez +âĿ¤ âĿ¤ +ro bb +ðŁĴ ¦ +lin ux +ten s +intellig ent +se ed +vo ter +s ler +pe aks +inter n +teen age +peninsu la +hand ling +ti e +cou sins +wen dy +me e +à¹Ģ ภ+din o +ðŁĴ ° +ðŁĺ ĥ +ze e +s bury +trage dy +b k +bo re +z in +war ns +idi ot +tou ching +contin ental +tac os +saf ari +wa shed +po dium +morri son +fore sts +c bc +al on +partic ular +be ads +inv ented +lo ch +li ghter +where ver +i de +docu ments +a we +k r +no where +min er +st it +ro x +contribu te +har dy +cl an +ob ject +ca it +ðŁĴķ ðŁĴķ +happ ier +vege tables +t art +g ag +nom inee +heav ily +pan ic +j d +there sa +at m +u ph +s fc +su ri +drin k +n al +re vel +k l +avoc ado +nom ination +ma donna +shar on +malcol m +control led +sh ers +revi val +legis lation +shoo ts +n in +comm entary +pro s +human rights +str anger +mit ch +pipel ine +leg ally +th u +gil bert +tol l +gran ted +gh s +ir anian +refre shing +du k +ab i +pri me +jose ph +mo sa +stati stics +produc tions +mer ry +pat el +sa x +human itarian +struc tures +e missions +town s +fre el +ster ing +rat ings +alle gedly +cab in +st l +w ade +fl yers +tri m +promis ing +z u +bal lot +compar ison +free ze +ou ter +great ness +as sign +snow y +r ale +tor ies +med iter +kno ck +consult ant +cincin nati +analy st +sc oo +je ws +appro xim +pu re +portra its +cy rus +ation al +lo ans +acqu is +el u +accep table +uni on +water color +ru st +batt les +per fu +seas onal +ser ial +mind set +ri ot +fel d +enni al +clo set +pri est +tan ks +int l +scre w +bu m +ab dul +ou x +expla ined +ric a +imag ing +law yers +bu ried +ãĥ»ãĥ» ãĥ» +ear l +âĢ ķ +l ton +resto red +stri pes +fo ss +de mands +ste aling +alex is +mun d +ak er +ur us +war dro +hu gs +gen re +e go +Ù Ħ +particip ated +bab es +ban quet +ti ous +he mi +ds b +lo st +milwau kee +jen ner +ge m +ou tra +lo ses +id i +re ps +ðŁİ § +regu lation +fla w +f ang +vibr ant +ram p +ra ins +well being +so viet +vie wers +de po +libr aries +bi go +ser y +g ill +de struction +co z +c x +bri dal +al ds +plan ted +amate ur +lu d +che ering +show cas +pro file +i u +ver tical +pack ers +wiz ard +ski p +s light +be au +air ways +mu ch +re ra +ðŁĮ Ĭ +ab sor +pati o +pack ages +s ells +ment ally +ðŁĺ ¢ +reyn olds +k are +tri bun +wal t +kn it +ta ste +sur rey +boun ce +cre ature +b are +bet ting +su re +mi ley +laugh s +al ore +cy n +t l +arti st +ann ah +war mer +dynam ics +lunch time +mariti me +vulner able +ðŁĴ ĥ +wol ver +dur ham +const antly +am in +si bl +: @ +bul let +k ach +angel o +wil der +doo m +desk top +law suit +k ca +hen derson +inv iting +bet ty +ta wards +ra fa +le aked +and i +ge ms +af l +vel o +mediter ran +pro be +to tten +steph anie +sn ation +com be +q s +over come +assas sin +ra v +fil ip +winni peg +sh il +determin ed +k as +ou tre +regre t +gui des +aa a +ðŁĺ Ī +wi ves +mani fe +er ly +sm y +sh ima +x ing +pix el +jac ob +ac commod +to y +on o +po o +ti er +an swe +ðŁĴ ģ +ro sa +le ase +bel ongs +th ar +eventu ally +nei ther +go a +ski ing +at ra +ag h +broad casting +f ury +py ram +d ice +volk swag +wom ens +provi der +bom bs +miss ile +whi p +d ick +nor we +back up +el der +mat ure +concer ts +gi ous +sque e +good morning +bra ves +^ _ +au ssie +lun a +mal es +he ck +for tn +rome o +steel ers +p n +pe er +re presents + « +kat y +migu el +requ ire +cha ins +l ur +immedi ate +ti mber +âĸ¶ ï¸ı +advoc acy +ex port +an z +tiff any +auth or +ðŁİ Ī +du des +chil ly +hi d +har m +bu g +mon ster +terri er +tu c +story telling +ta k +in ti +immigr ants +b is +reach es +com passion +john ny +contribu tions +ðŁIJ ¶ +mechan ical +impre ssion +ran ks +ko be +men ting +bloss om +pab lo +buil der +bom bing +tw el +sul livan +om o +pe te +de mi +ku dos +w bb +t gif +mass ach +neighb or +che fs +eng ines +pun e +ga ined +phan tom +s days +ext end +gr an +cent ers +jac qu +dat asci +sleep y +el vis +answe red +s lot +con y +flexi ble +ti ally +le tics +% , +andre ws +si ble +mom ma +vin o +do x +invit ational +twil ight +j ade +ill ery +joh ns +f ou +p v +-- -> +break down +billi on +prin ter +mon d +c bc +mag gie +legi on +du b +kur t +po or +paren ting +regi ons +bikin i +be ware +si onal +au burn +kid ding +amp les +sp an +con tempor +c ic +ha bits +ak o +pre fe +bud dies +it z +em ily +person nel +moun tain +ver sus +ðŁĺ ¬ +ear ning +s ink +dar i +u u +s win +i ster +bru tal +n ac +kat a +clo th +am and +ðŁĶ Ĺ +ne o +alu min +week ends +nebra ska +co des +delay ed +brun o +pro ven +in c +i ght +fl an +or o +lam bert +regu lat +w f +massach use +kardashi an +bern ard +fi esta +volcan o +grand pa +anc a +d re +st itu +mean ing +fo am +au ck +at ed +r l +hot el +pers ons +dy nasty +ell or +ma i +am ne +sty ling +avi er +e g +vege tarian +, âĢ¦ +foun ders +sta in +g d +cy cles +sky line +trac tor +exi sts +tra l +kid ney +mar il +inst ag +se tte +addic t +tri angle +flash back +controversi al +z on +p ins +i as +tr ay +town ship +deleg ates +sp am +h ms +cr ane +peop les +o lo +fac tion +but es +on ica +deleg ation +new profile +eli er +mc a +w and +g ely +losange les +ber ke +ti ve +dis rup +zz a +cas a +jor dan +ford shire +ga thered +ic hi +atten dees +à¸Ń ภ+pe ppers +co in +bour bon +ern ity +ro tary +behavi our +jere my +team work +compli ance +tre mend +ðŁĩ § +bu hari +cam bo +bu yers +ha gen +bu ds +bay ern +mon te +sm ells +an za +ath lon +descri bed +work force +gi ving +ap i +invest ments +da il +sel ena +datab ase +th um +mor tal +stu dent +bu yer +do ver +gar ten +att le +loy alty +gen oci +holo cau +theat ers +ru ling +ven us +pat ent +ch un +ab by +awa ke +mass acre +bang alore +break ing +simm ons +ju sti +hal e +ed chat +gg les +haw k +mar king +head lines +stro m +co ve +breath taking +med als +hair cut +christ ine +tele graph +gujar at +ju ra +can e +sho re +propag anda +mu eller +.... .... +sa vi +stom ach +thro ws +ta b +war m +j ong +reno wned +hi r +ra is +mush rooms +guaran teed +bo a +m j +revolu tionary +certi fication +bru ins +jo in +w es +pas sport +c g +sex u +cap able +w v +ton es +jac kets +ac compan +spin ach +fore ver +bla ir +wat ts +g l +cou ples +prairi e +newprofile pic +logi stics +massachuse tts +jagu ar +o id +we al +under water +mo z +y i +ma ths +myan mar +pre ps +suffe red +tr ace +wal i +ah hh +bor g +st itch +cu lin +real ise +infe ction +discrimin ation +sh ame +an kle +hu mid +y t +brac ket +tru ck +tri u +ea ster +commun ity +post card +invol ving +ty ler +car amel +over view +ex amples +integr ity +base ment +instru ments +ani um +at us +gh er +laun dry +achi eve +gen eva +pr icing +hyder abad +beli ef +me ta +j aw +accoun ting +lead er +cristi ano +cou ture +cy p +vis ed +, ,, +k nu +h ick +break er +br am +ra b +mo or +ham as +gradu ating +pupp ies +ak h +ta h +ach es +ri e +op ini +g ta +re ign +tra gic +re ver +p ill +pine apple +tou ches +da re +le ys +il o +inter iors +sc outs +bar t +en zie +don o +bro ck +christi ans +ense mble + · +cine mas +new port +air line +win ston +le igh +cont ents +pre scri +ur ge +tr out +fic ally +il ia +sub si +are r +âļ¾ ï¸ı +w ounded +ðŁĻ Ĥ +pe pper +ðŁĴ ŀ +fit ted +af f +re sur +thursday thoughts +z ero +archae ology +di v +je e +i on +awa iting +co zy +beauti es +bal d +dat a +gri zz +stal k +kin ds +cle ared +jess ic +regu lar +ali ens +plac e +bo s +bi zar +thisi s +ðŁĴ Ģ +totten ham +ma fia +s lam +ari ana +car roll +back pack +care y +uni v +r g +pe p +dig it +tatt oos +ag on +volunte ering +diffe ren +consu mption +ka thr +head phones +t shirt +o b +ele ment +re tail +sh ru +al gori +contain er +consci ous +fi l +com ing +ra sh +u rope +def ine +gi or +femini st +flow ing +rout es +gl aci +fer t +somer set +ant es +twee ps +$ $ +h our +endange red +year sof +ro h +po pped +bac king +ba sil +bra ke +mon aco +lgbt q +pra gue +ut ility +cas si +gate way +haun ted +sch ul +ðŁİ µ +shou ld +walking dead +comple ting +dann y +montgom ery +pengu in +ss i +mer chandi +ðŁij ij +chur ch +h ates +cap tain +brea thing +ce t +fair ly +approach es +compan ion +surpri sing +kany e +pe y +hin di +targe ted +lor ds +de ut +di gging +ger man +ru t +ener gy +close st +y un +apo logi +ภ± +s ack +ru p +dd y +port al +d ough +b ats +ðŁĵ ° +at ur +graph er +pi res +mo tors +ðŁĮ ¹ +j c +dan g +tu k +clu e +us c +pag e +d less +bro ws +ju s +ad ing +re marks +oo m +car dio +ste fan +arm strong +âĢ¢ âĢ¢ +ni est +belgi an +bi op +so y +lo f +í ĥ +q t +flashback friday +ce e +ģ ภ+wre ck +mar ines +amend ment +wardro be +vo y +bur ned +guit ars +ra inf +li fel +ssi l +oun ce +exter nal +c key +me sh +she ikh +inv itation +sugge sti +pop corn +phenomen al +an onymous +tun a +chic ago +o val +del y +loc als +( & +pro f +no vel +fin der +spar ks +la ven +in fu +nic ks +qu ant +ra e +exe c +dist ingui +st ances +mu tual +sh al +unve ils +edmon ton +zan ia +a dio +vie wer +brad ford +audit orium +qu is +re act +htt p +l ero +chee ky +impac ts +ta k +ed t +desper ate +t ay +ì Ħ +sett le +bar gain +resu me +un ite +thro wn +ke st +se ys +mar ching +am it +decl ine +sch ar +me tr +stan ford +lin ke +ber ra +dol ls +rug by +jam i +b or +road trip +dino saur +mi k +sun der +re m +b k +over seas +nau ghty +imple mentation +iam srk +lun cheon +fir ing +mi ami +pere z +the e +z on +gi fted +con version +ceram ic +¡ ï¸ı +pe dro +ì Ĩ +v ick +! @ +he ed +si d +b w +docu ment +pl un +gr ants +fant asy +predic tions +vali d +car ved +gradu ated +ðŁijį ðŁı» +nation ally +ch y +af l +re sso +blan k +ri vals +j ig +e ties +om ics +une mp +b ound +sk o +inspec tion +par al +high s +cri sp +b ans +ob a +[ @ +co spla +costu mes +rec all +mou th +ni gel +b ts +ter a +ko v +do cs +west minster +dic t +gra vity +kar i +ro gue +t ted +war k +ida ho +w end +aw i +queen sland +proce sses +cli ffe +m ick +com pens +op ol +the y +cl ari +wiki pedia +salman khan +haz ard +pre ston +swee test +pd f +che es +tr ilo +south africa +bur nt +( $ +con tain +t p +sub mitted +sound cloud +at u +re z +word press +corru pt +n f +ma ker +í ķ +par as +adv ent +ri al +ca fe +fo ssil +!!!! !!! +co ws +c j +sp ur +institu tions +land mark +ent it +re ut +h is +alz heim +we mb +regg ae +mo squ +st at +identi fied +deal er +re am +re land +ten sion +ðŁĩ © +wra pping +deep er +fr at +red dit +ar is +moroc co +.. " +b low +ma pping +pri orities +ing a +swa p +re wards +conspir acy +creati ve +c j +congre ssional +vau lt +ple x +sophom ore +shad ow +ele ss +ðŁĺ ħ +dar ts +aldu b +anno ying +pro ps +n as +alumin um +h bo +offen se +j ill +oni ons +la ur +ta e +har dest +sh ro +ga ining +meas ure +ed tech +cyp rus +tar a +ang eli +car lo +go on +all i +im plic +ju pit +resil ience +ha il +bal anced +) ... +joy ce +gr a +th eli +defin ed +shi pped +main ly +min a +l m +sac ri +o ber +p im +claim ing +ent ers +co rey +bo k +cri ed +cool ing +dani elle +pharmac y +thor ough +ca ke +k lo +outre ach +z ens +digital marketing +val ent +sn p +her b +mr w +caf é +cap tures +no tre +triu mph +pan cakes +cu mber +spi ke +d ation +bi gg +sp er +crit ical +am al +too th +foun ding +a stro +' # +quan tum +th ames +un c +pri de +air bus +kno cked +un defeated +mediterran ean +cal cu +clo wn +sens or +ham mer +for give +cu shi +ber ry +maje stic +elec t +polit an +g ta +k ari +bur ke +sea hawks +volkswag en +re i +landsc apes +cas u +grand father +list ened +/ / +star trek +rainf all +fur ry +vi er +star k +rif le +ff a +leg es +hillary clinton +min us +correc tly +architec tural +pre ce +up side +box er +ðŁĻĮ ðŁı¼ +is ai +de t +pro vo +tis sue +spoo ky +ve led +re con +prospec ts +que bec +âļ « +ig no +anat omy +shap es +w p +p interest +hor e +an es +pick up +ti p +pra desh +hu gh +co e +po k +gram my +well ington +sti gate +ri gh +lea p +king ston +scen ic +go sh +v ani +au g +s ary +zi er +bure au +lin son +con te +fra gr +all an +g aw +lan a +colli sion +surve ill +ren ais +ar range +s ali +do in +br ance +bren dan +our se +in coming +suspen sion +à ´ +l la +educ ators +in tri +da e +bio graphy +bul gar +villa in +go thic +rw anda +e w +may or +meet up +democr at +mor gan +su dden +te sco +car rot +bom ber +mck in +re ne +fun day +agricul tural +haha h +show time +form ing +col a +scor pi +quo te +po ppy +s life +d az +tu b +ne n +mo t +ðŁĺ » +s ore +elder ly +o ve +skin ny +um i +anc o +man ship +we re +g v +k ah +fol ding +ne at +samanth a +dan ish +uk rain +humid ity +nu tri +jak arta +cand les +oooo oooo +at ile +streng th +i bra +bap ti +charle ston +fr ames +girl s +clear ing +glu ten +# # +super natural +ju bi +ph one +he in +dr un +le ak +invest or +y er +dom ain +ball room +mi sh +app li +off shore +bla ze +dor o +âĺķ ï¸ı +win ery +shar if +ad ore +n ir +saf er +si gh +as cri +strong ly +trac y +ck er +ol l +faith ful +ey ed +deli ghtful +vis m +karnat aka +tit an +wh ar +jer seys +re fur +heav en +gri p +pan ama +pre li +glu ten +o dd +cont ent +pon ti +tion ing +e commerce +feder ation +flaw less +ge ar +ti res +by r +pol ice +cu ban +tri butes +tic ul +chur ches +nur sery +di aries +muse ums +snapp ed +i van +wi ght +touri sts +ramad an +t rent +prophe t +won dered +focu sing +hi d +ic ons +i q +ambul ance +pi st +fun niest +time less +sr ilan +bu ys +ki ds +colour ful +a shi +ch ir +mu m +ðŁĵ ļ +let ter +x en +reut ers +pre serve +in ting +ste p +fu ji +uni ver +i u +show down +po ems +surveill ance +suspec ted +ta e +sol ving +tom b +mother sday +car pen +recru it +pil ots +bro c +mix ing +fri days +ty r +represent atives +tra pped +abdu l +free style +clu ster +âļ łï¸ı +k d +sk ill +pit t +ex o +commer ci +muse um +loc ally +g ina +no bel +immun e +fr ac +cap su +main ed +attemp ts +bull dog +be spoke +sing ers +sp elling +seg ment +nat ures +tic k +lip stick +clean er +gett able +preci sion +âĢ¼ ï¸ı +th ood +re ef +no pe +bill y +di gi +mu si +ri val +figu red +tal ity +sun ny +ber k +aw ww +awa its +un real +co pen +asy lum +ex otic +bu en +mo ck +en able +arch y +fr a +pla stic +al mond +amp li +displa ys +abbo tt +s me +x p +ðŁĻ ĥ +graph ic +i ved +mar a +cau tion +lea ks +en berg +ul u +unic orn +cann on +appren tic +ðŁĺĺ ðŁĺĺ +b ball +wil low +at ics +am as +manufac turer +campaig ns +port ers +flo ors +l su +ty pe +ke j +honor ary +it im +to le +min ecraft +d x +ma sh +ri o +consequ ences +ron ald +go ssi +suffol k +mu se +r bi +live music +i van +ðŁİ ¤ +le u +patri ot +man it +lan ca +home decor +de ar +sig ma +ti de +str ings +v ita +sequ el +try na +inve stigate +bor is +ve gan +barri er +mind fulness +web b +hu stle +in da +tan zania +str ay +tex as +c ag +diagno sis +wom an +g w +ob session +l ative +nu fc +fl ynn +moment um +sof a +wal d +vege table +tu cker +supp er +se ab +ar ro +se ag +ven ting +counc ill +sp lat +cal cul +.. # +com fy +odi sha +sto pp +war fare +ca es +à ¨ +co y +price less +in sec +ðŁĺ Ľ +contro ls +empower ment +datasci ence +per pe +gen ic +e res +tru deau +man o +sla very +expand ing +ma he +fa iling +s aga +photograph s +cre st +re on +surf ing +hi e +ðŁį Ģ +ja e +fel lows +south ampton +sol om +ce ster +tab ility +hor n +se ct +he e +cole man +at las +explo rer +consul tation +copy right +organi zing +den ied +mon keys +noo dles +br is +fl or +dou gh +bon ds +sho cked +eco system +care fully +w m +apart ments +cur ve +san diego +must ard +comm en +cere mon +e ch +ru th +ðŁĻĮ ðŁı» +hawa i +fil med +te ar +as ingly +ca ir +wat t +instru ment +ou tta +ye ol +river side +ë ° +. : +nor wich +alo g +migr ants +new man +ri de +spr ink +targe ting +beli eve +tor ch +reflec ts +per mission +ff man +ene mies +bas ics +se ized +sun days +le i +hass an +en do +h c +st ad +le ments +kk kk +nan o +shar k +man a +on ic +treat ments +ear ly +collabor ative +shu ttle +bran ches +mis ses +mained cm +ap ers +ky le +carri e +leis ure +sh et +bir ding +adv ances +ðŁĵ Ŀ +popu lar +di ane +a be +re war +neigh bour +k pop +remem brance +play ground +ru b +krish na +e bola +inqu iry +ep a +lu min +organ isation +abra ham +norm ally +pre ten +jan et +w t +ðŁĴ İ +encoura ging +a stic +bu mp +syd ney +s z +ss ss +gar rett +ðŁĵ » +consul ting +roman ia +spo tting +chanc ellor +ar ma +presti gious +ðĿ IJ +t ad +cry st +compe tit +rati o +cat aly +bro w +j ur +vi king +commu te +y day +la yers +du mb +esc al +genoci de +f ill +gu pta +ste pping +se i +fo to +wild cats +col i +projec t +ear nings +st r +ge ons +comple tion +b m +decor ated +craw ford +af ghan +sc are +visi bility +hi b +direc tion +stro ll +christ ina +alter nate +cl are +sty list +be hold +s ance +leop ard +acqui red +narr ative +ash i +the a +?? ?? +pe as +at ch +sli des +le en +renew able +eng lish +qu ir +co aster +r x +fo ols +match day +mis m +amaz ing +z ig +ke ting +won t +to wel +di ab +sta ke +n m +mel t +e than +gra pe +polit ician +sm en +í ĺ +re o +wedd ings +cat cher +or acle +me mo +ðŁĮ ´ +ec k +rob bie +norwe gian +oper ator +am or +se wing +ju l +x ie +u v +fif ty +me ga +tatt oo +liber als +u pri +traffic king +richard son +su v +ki p +mess y +tremend ous +gl ou +cour tney +la d +stere o +my ers +i dio +^_ ^ +man ning +dy e +w d +thr one +jun k +as u +provin cial +k ook +wr c +fine art +hamp shire +renais sance +b red +fall out +s j +sn l +al am +tor ture +fy i +sh ines +pa w +ch ar +hen ry +c row +aci ous +di an +pa ige +ba re +stock holm +scen ery +ðŁĩ · +jef frey +pu sh +decor ation +ne d +cu te +brig ade +laven der +inv ites +e sports +vo ir +dri ed +tran spl +sur geon +no vels +pul ls +son y +lun ar +man e +i vy +fru str +dor set +sa i +tor res +ssi on +shut down +suggesti ons +writ ing +e o +battle field +u ga +ðŁIJ ¾ +vac u +spl ac +g it +u g +high land +% ) +mer maid +sacram ento +ta ils +p w +ka h +t ell +enh anced +ì ķ +auck land +cru el +ðŁ¤ © +au dre +sail or +gram mar +g love +de on +infl am +fresh ly +k ell +zi p +christi e +mil d +di xon +instru ctor +g ence +ãħ ł +sub jec +constitu tional +crow ds +in visible +ru ins +da k +si p +pla que +p ouring +comple x +z ine +ste ad +f let +trans mission +lo way +ar un +incre asingly +au d +transp aren +cro wned +sc oun +blizz ard +lux u +fi ers +achieve ments +hun ters +rock ed +bas in +vio let +pro ves +achiev ing +pro sper +se ga +flo at +vi an +xi v +pol ic +tur a +approxim ately +wander lust +keep ers +geta way +co d +pol is +br yan +col ts +tal ents +yo gur +gluten free +wri st +gr y +cze ch +ðŁİ Ī +ev ille +ðŁı Ī +to x +dani els +am er +bi ds +weare one +me tab +g t +boy z +pd x +pos session +pu shed +shr ine +reali stic +tri gger +na vi +ru mors +n af +jen kins +tr un +comm uni +Ã Ĺ +gam ers +arm or +moham med +bal cony +y ah +stron gest +rhy thm +unfor gettable +k p +ho bb +custo dy +greg or +r ita +aes thetic +il ation +sponsor ing +n ay +kid napp +sh s +ra jas +me g +signific antly +butt ons +la c +ver sions +essenti als +opini ons +k ro +d printing +wi dely +d k +ur an +y al +reque sted +c n +cur ric +plu m +gr un +v m +dev on +m yo +rel ation +juvent us +rou ge +min ority +min es +jupit er +n ine +oxy gen +fran kie +une sco +fab ric +disgu sting +sal man +dete ction +lan ka +d ac +ðŁĩ« ðŁĩ· +argu ment +shel ves +cel tics +rober to +pi gs +he dge +fau l +pow ering +butter flies +fi r +re make +att i +com o +emp ha +kend all +poke mon +se ating +d ans +bald win +ðŁij » +lesli e +one direction +ti mber +im an +fon t +e der +di on +ste ph +for mat +gre gory +pro p +he x +ru in +sor y +inf er +n aw +bar ak +sd gs +kar ao +lu sh +v ander +end ent +g is +a fro +soc cer +ay an +t uni +lun g +da yof +alex a +mar ath +addic ted +ag ile +hy gi +light weight +ì § +mand ela +jo ey +anc y +hu m +bi r +memor ial +jim in +ging er +v ak +jav ascri +cro ps +orig ins +d ari +pi per +im port +aggre ssive +predic tion +re pairs +cr acker +voy age +ni ke +mu mmy +linke din +country side +bor der +gla ss +per t +s als +sho e +autograph ed +wal nut +colle gi +sal ary +pa iring +ðŁĮ ¸ +cath ol +swee the +defe ats +streng then +roof top +impro vements +barri ers +ur u +t ally +ru led +ðŁĨ ļ +nai ja +emo ji +per cent +gi o +pro bs +on ce +adm its +pa ths +li ar +day tona +pe ters +cal i +cal li +mu g +o sa +ap h +ab y +hy de +eth nic +pla ins +ol f +haha hahaha +holi c +?! ?! +su bli +bl acks +mo t +gh ton +lo vin +b rent +bar u +l ati +de w +ate au +q a +pain ful +bu sters +st atic +ðŁĩ¨ðŁĩ ¦ +note book +out fits +si es +r f +floo ds +Ñ Ģ +thro at +su ici +ro vers +beng al +pre pares +blo g +mini ature +Ø ¨ +am phi +com b +r sp +in timate +green e +Ì ĩ +al tar +surg ical +ves sel +... ? +gav in +g ator +threat ened +z ar +rob bery +di er +promo ted +y g +x s +su bs +inter viewing +threat ening +do zen +me ado +water fall +nintendo switch +cal um +mini sters +dro p +univers ities +war ned +tac tics +ðŁĩ ² +refu se +ad ju +v ast +ðŁĺ ´ +mc fc +lib ya +no filter +distribu ted +re ser +ron nie +de co +javascri pt +mon k +intere sts +fle x +mar tha +sti es +oo d +ðŁ¤£ ðŁ¤£ +e un +b ali +g omez +sti mul +moder ate +d ity +ir is +stra w +consist ent +direc tions +adop t +sal sa +cro o +reco vered +black friday +lan caster +accep t +weareone exo +buil ds +free man +air plane +diti on +bel ong +jam ie +pit ching +li f +om in +cri spy +pre pping +ve g +chan g +accompli shed +graci as +dolph in +elec tor +culin ary +super bowl +wal a +pur suit +black berry +be an +cardin al +pro ved +immigr ant +stric tly +holocau st +pass age +ha us +cou p +pur se +har ass +< < +le ed +ado be +st ad +legis lat +par ked +pri yan +sil va +kri st +s the +fun ky +ig a +sett lement +ph s +t mrw +stre ssed +hun t +ho ckey +treas ures +cham bers +ol u +hu t +mar ley +tex ture +wilder ness +mm ing +poten tially +om aha +ju dy +to es +spo iler +distingui shed +feli x +ah u +recommend ations +zom bies +hit ler +tri ple +colla pse +motiv ated +ulti mat +gg ling +so y +ci gar +fo ren +vine yard +gl itter +fin dings +colon ial +hun ter +eri k +den s +beet le +lot te +sub tle +s matter +tru sted +experim ental +nam ents +ðŁĺ Ĩ +regi on +acquis ition +bre eding +quarter back +am reading +oo td +ru de +initi atives +st out +hy ung +out come +al fred +mic s +exper tise +bacter ia +pengu ins +jump er +valen cia +bar k +ing day +sell ers +contrac ts +hou ston +commissi oned +adap tation +swan sea +santi ago +common wealth +ju dging +sub mission +sco rer +tom my +ñ o +ex quis +fil ing +explan ation +alli son +wemb ley +ri dge +chev y +san tos +own ership +cogn itive +favour ites +sh ed +phil anthro +dele ted +go dd +s nor +gui delines +ff ing +je ep +cli ps +sw amp +an or +guil d +bol ton +spring field +munici pal +goal keeper +ye on +ðŁĺįðŁĺį ðŁĺįðŁĺį +ãħĭ ãħĭ +water front +gra ve +contempor ary +ar ity +ÃŃ a +sle eps +sy rup +al am +pi re +co yo +moto gp +ty son +kej ri +cir cul +sing ly +cr unch +complic ated +nostal gia +k op +mo ve +k ale +mac ro +mid west +h ans +tri bal +nu de +௠į +bey once +congratul ate +cat er +leagu e +ðŁĻ Ĭ +la dder +cra shed +tech nic +karao ke +harass ment +ro ts +experi encing +kri sten +ðŁĩ ³ +ðŁ¤ Ĺ +reflec tions +guin ness +illustr ator +ðŁĻı ðŁı» +cen ter +nar row +comm ons +regul ations +Ù Ĩ +har m +cro ft +cu ssion +hong kong +st ical +intern ship +zo e +cho p +hoo ds +estim ated +batter ies +berke ley +smooth ie +shau n +cro s +~ ~ +cam pe +hu mp +b g +proto type +cl ick +shaw n +re viewed +tem pl +p f +jed i +blo gs +ray mond +as th +ba h +av ail +scot ch +leaf s +nik ki +to k +hol low +ur ges +of t +un like +lat in +u e +cat ering +mil i +alter nati +ma ver +Ð ¸ +ag le +pre order +lu x +cu cu +ðŁijı ðŁijı +t art +âĿ¤âĿ¤ âĿ¤ +arab ic +rapi dly +ar rang +all en +travel tuesday +pa ws +flo ws +st ability +flu id +ca pp +can berra +uu uu +sp ani +demon stration +m la +plac ement +m w +presi dents +awe som +bever ly +ani st +ne al +father sday +referen dum +la hore +o aks +deb bie +half way +gho sts +de bor +matthe ws +fi at +t fw +pre sen +rob i +de d +bro ck +laugh ed +am ounts +bam boo +kinder garten +eat en +mtv hottest +break out +u sic +fra ser +legis lative +p ang +modu le +sam my +go ver +ear ns +expe dition +gar h +concep ts +char lie +la va +bachel or +veg gies +deter mine +el lie +un locked +fru it +dal la +cou pe +wash ington +depo sit +iv ory +pau la +chic ag +gu cci +ðŁİ ĥ +cul tiv +pier ce +li fted +stu mb +re cover +musc les +conduc ting +cb s +mcla ren +sophi a +cel lu +oce ans +up loaded +game play +mal dives +kim ber +avo i +rac er +ca ine +cav s +h ana +li ga +ra ven +inter vention +inaugur ation +oo h +at traction +merchandi se +tune in +li king +juni ors +int ended +att acking +aqu arium +i wd +comp onents +sur ing +cent u +yogur t +ðŁı ĥ +show room +op tical +ty our +ju dge +yi eld +an to +pl c +transparen cy +recy cled +chi ef +ar om +ambassad ors +plan et +âĿĦ ï¸ı +om ed +vaness a +cour t +mar gar +hal ey +v r +reg ina +pd ates +hi span +live stream +âģ £ +ya hoo +gal la +secu red +w ir +bene ath +off l +n il +am b +ye g +out let +u te +pe ep +lind say +bent ley +... ! +he el +trilo gy +vo s +ty re +there fore +tor onto +ab i +simp li +ja e +exten sive +eleph ants +s or +orient ation +im peach +re play +constru cted +peter son +pa is +por ted +custom s +colla p +ad u +high lands +sal em +shel by +ko vic +stra in +ro sie +sen ators +snap s +bo bb +suz uki +bla des +k p +lo lo +gener ate +si ght +ma e +struc tural +predic t +jump ed +ah mad +sun g +just ice +gla m +vol vo +jubi lee +de tention +lo sses +pu ri +every time +Ð ° +ra o +ed ge +li mer +rese mb +har old +re tri +sacri fic +surpri ses +am c +srilan ka +bar bie +men s +fin n +ag s +ukrain ian +em brac +î IJ +flav ors +hom er +lau re +ou th +pr iced +ver de +fir m +ah s +cu b +tre y +par anor +pro fit +in dv +who a +har sh +al ot +crit ics +hu bby +fi gur +gi ra +ca stro +chan el +in put +origin als +ten ant +yy yy +ture rs +lincol n +co on +lear n +ch ou +ac are +o les +din er +hy p +bizar re +mc r +let sgo +decor ating +ðŁĮ İ +al ison +ar vin +f d +reha b +mccar thy +lot tery +da h +minne apolis +eli gible +diagno sed +emer ald +destin ations +s ans +or y +bla zers +n v +ba il +digital art +no c +mal ta +sol ar +pi pes +alleg ations +no ck +po pe +bri d +premi er +n x +present ations +ef a +bo ws +val ve +opp onent +Į ë +visu al +ing le +cate gor +e ter +po is +dan i +at tract +neu tral +th ene +cra shes +fred die +ut ili +c st +awak ening +slo ven +quali fy +pro of +fair y +le v +fre ight +enjo ys +cup cake +flav our +â ķ +protec tive +ðŁijı ðŁı» +is u +ad mir +h mmm +continu ous +ai res +rap tors +showcas ing +y uk +pa ste +follow er +instru ctions +sp ru +@ __ +the o +debu ts +ve tte +sto w +es of +ach ed +sul tan +sand wich +som alia +franc o +car ne +flu ffy +al pine +jas mine +he ated +viol in +ple ss +divor ce +per former +phi es +port sm +dar a +kir by +lo p +chill i +for th +sky pe +ðŁĩ®ðŁĩ ¹ +celebr ities +ed y +ve e +po ison +ey el +gra bs +ssi c +un o +wester n +rail road +am er +numer ous +s v +fo w +fi st +âĢ ĭ +reque sts +mar tial +em my +accept ance +lau ra +ภ´ +er up +hyun dai +out lander +u tt +wrest le +esp resso +demand ing +g dp +geo graphy +sas kat +tro ll +confe der +su es +se m +be ts +t ful +to sh +teach es +col oured +gal way +mac y +dis orders +bb cra +at em +fen der +lit ter +e sh +provi ders +renov ation +nomin ate +ps g +nomin ations +jen na +shar p +some day +z ur +bra ins +che shire +pre y +hu go + ¿ +to ken +r v +car r +tac tical +zel da +kay la +fern ando +photograph ers +j our +umb rella +woo dy +congress man +du mp +le vy +ju an +d azz +sign als +la in +an u +mic hel +por ch +al den +sibl ings +y ale +pe el +sw ick +gg in +ll c +k ale +s con +il d +pat reon +re el +qu in +wit t +mar ty +moo dy +ton i +der y +g ators +speci fically +dd in +ly on +tr ick +meado ws +p j +bor gh +vi k +tu r +bron x +pu ff +lan tern +ðŁ¤ ¦ +g ently +be stie +fac t +refu sed +fas ci +mp y +ðŁĶ µ +cross over +mead ow +indian apolis +duc ation +sle y +loo m +mix er +new music +film maker +prosper ity +li m +week end +cre amy +neu tr +lu ther +h v +nor thern +tw o +h ra +cat ches +appear ances +ha bit +kitt ens +n v +illa c +inf an +regar dless +liz ard +dun k +cur tain +ac om +in tu +ve z +e min +fl ats +calend ars +em power +ru ined +hun gary +vi d +we x +u lum +aber deen +o sa +k t +ma ssi +se emed +s den +' ? +tele phone +de fi +insp ires +me ow +z ones +bl ind +pl y +tuc son +advent ure +ge d +oy ster +ðŁijıðŁijı ðŁijı +out put +tt t +metal lic +sma sh +ucl a +sco ts +perfe ct +lu cy +regular ly +sp ic +rel ative +ath ers +mis e +batt ling +deci des +mat a +occu pied +random ly +cat softwitter +gi an +ball y +al ties +al lies +im men +sy rac +ðŁĴľ ðŁĴľ +l lan +au r +k ut +lam ar +affe cts +n ra +star war +ðŁ¤ ĺ +sc ram +en chan +pro cess +luxu rious +ar ray +sher lock +comp ati +dor f +stre ss +m su +s with +sal a +sof instagram +fo il +under stood +qu ay +r p +c ade +ja w +en ab +en coun +ðŁİī : +do ck +satur n +mu ll +lay out +ra rely +happ ily +fix ture +or ph +over looking +her bs +m itt +pil lar +nol an +pe tty +str y +u i +mu k +o res +o vers +á µ +re creation +we sley +ri t +kejri wal +sto cking +g v +subscri bers +moo se +ma e +ber t +opp re +assign ment +u ro +high lighting +cal vin +we igh +cambo dia +av on +ke m +dis abilities +read y +char gers +p ads +iz ing +illi an +tru ste +col leges +associ ates +alban y +mil ton +cr on +bu r +har dly +si ghts +anti ques +e cho +surpri singly +ha iti +cap t +ph p +op io +ine quality +equ al +ken y +sch mid +autograph s +ren t +qu er +cit rus +challeng ed +te c +epi de +fe st +z hou +li me +citizen ship +cry stal +convin ced +mess enger +copen hagen +âĿĹ ï¸ı +war ran +develop ments +ï¸ı âĥ£ +fore x +hi ro +sne akers +xi de +vi va +stere o +bat ting +ss el +ho st +beng al +critic ism +q c +cr un +attemp ted +ry e +determin ation +cre ations +d read +label s +pos se +anc er +joh an +si ster +partner ships +les bian +k st +guaran tee +bar o +fix ing +ma son +m ous +chem icals +t less +bio diversity +par o +bhar at +ac ol +refu ge +en te +t iti +dys sey +respon ds +lef to +in er +se vel +rahu l +ol ine +frank fur +cho reo +enjoy able +c to +strugg les +wood land +heavy weight +gen s +rece p +ac cred +ðŁĺ ¡ +trans formed +list en +at op +n k +sur ge +be re +gover nor +prison ers +clau de +t ill +mu lator +emo tion +water loo +star t +ðŁĩ º +clean ed +grand mother +fear less +afric an +astron omy +ðŁı ģ +ภĻ +the world +su itable +anth ony +k and +tt en +meaning ful +disc lo +jaco bs +à ¸ +tom linson +ghe tti +ty pho +sub stan +as co +te k +nag ar +mu d +am on +vacc ine +f ty +fle sh +no el +infl ation +portu gue +glam our +tra m +v re +te qu +roun dup +w yn +rejec ted +mosa ic +si ghting +cal f +o ta +com position +go pro +gonz ale +e ed +b ard +tu e +effec tively +we en +al to +ri bs +rel ate +thir sty +fu rious +di m +ch ard +perfu me +s ny +chur chill +k of +master class +wa ve +ðŁĶ µ +er in +own s +to be +sk illed +te m +go f +en i +tor i +cra zy +l ick +resi stant +ici al +ag ar +! : +g ali +del aware +bl itz +koh li +pu ck +avail ability +hi malay +influ ential +cro chet +victor i +read ing +ho bby +vie t +j as +en gra +sk ul +ðŁĩ² ðŁĩ +educ ate +tech no +distric ts +blu es +se tt +seven th +lear ns +ee ee +apocaly pse +hang out +cru el +mu tu +bru h +hel en +she er +c tion +kle in +tex ans +ce real +sh ine +ne red +gra s +am bro +f ella +hin du +matthe w +li ma +mir anda +je wel +so ho +euro vision +neighb ours +chand ler +be sides +ðŁ¥ ° +ast ros +thu mbs +ren ault +ra ve +hi red +ðŁĸ ¤ +it ary +z or +bla zer +k ine +ea u +kat y +dc comics +pe c +ro dgers +water proof +kill ers +super int +pre serv +as so +brew ers +promo tional +sc am +villa ges +sket ches +ju icy +for life +au dit +so lo +fundam ental +len e +philipp ine +t end +conserv atives +sponsor ship +dd le +a ine +h tc +os i +hul k +w af +ภĻ +evalu ation +ant ine +sle e +robert son +roo sevel +ag i +sophi stic +emplo yers +bubb les +ko wski +inter action +sh u +bou le +ic an +j are +han k +leg itim +k nicks +kar ma +recei ver +per ks +u h +sta ir +sun i +labor atory +gra ves +voc als +oo t +c ture +thri ve +tic o +ãĥ ³ +b w +carto ons +mcdon alds +dra w +y ung +pl er +li d +eth ical +groo ve +ent a +international womensday +pat ron +wor ries +ðŁİ ħ +ðŁij ĭ +ka therine +di az +tor i +bach chan +tru st +min eral +ic om +buil ders +bor n +col oring +lat te +ca se +revolu tion +tra der +ox id +chi pot +inst antly +sou thern +se hun +pro b +her nandez +lis bon +hu awe +p ong +me a +ro oney +wheel chair +ke en +be tt +cor in +regulat ory +di splac +ka ren +sch em +sun sets +wh ales +remin is +he p +hi de +mar cel +pand ora +do yle +th fc +ot to +no kia +trans gender +ko v +hawai ian +sha ve +so vere +exc er +nick i +pu g +st or +ro th +wee t +leg al +dig nity +po w +hom age +ðŁĩ³ ðŁĩ +s re +can on +la x +wo ah +quart z +ñ a +gree ting +flick r +nai robi +advoc ates +an c +vi i +eu gene +th ra +c re +el an +pen sion +th letics +ton i +re agan +x v +sto re +ben ch +har lem +todd ler +sent enced +âĻ¥ ï¸ı +glob ally +che aper +u f +ma m +nic o +ik u +tho u +ni st +dam i +th ala +rho des +sal e +bow ls +â Ī +las vegas +sanc tions +adm ire +mat ched +un able +travel er +ele ven +straw berries +âĢĶâĢĶ âĢĶâĢĶ +stu dio +jac ques +im s +valu ed +s no +cheese cake +n xt +e os +s x +f x +ton ic +hat ch +chic ks +gra ds +hand ic +r ory +as p +ri pped +denti st +n en +lu fc +âľ Ĭ +di ge +hop kins +sher man +f da +for all +ash ley +str and +h y +liqu or +buffe t +ess ence +phar ma +suri ya +ðŁĴĻ ðŁĴĻ +festi vals +z an +re fresh +pur ple +uni forms +kenne th += ) +as an +hel sin +transform ers +k ali +person alized +chal k +bo bby +â Į +the mes +depar ture +prin t +illustr ations +qui et +agre es +gri ff +Ø ³ +m iti +toge ther +conven ience +ab ar +car lo +turt les +info sec +some what +ar lington +scholar ships +emir ates +mu ms +st ella +auton om +fe ather +g ore +nom inees +fragr ance +Ñ Ĥ +w ong +thea stern +gr e +z illa +is i +bump er +go o +do zens +ab duc +âļª ï¸ı +o ils +don ors +sil icon +i pod +fortn ite +ðŁĴ ¨ +tor o +spark ling +consci ousness +pal a +nu m +moun ted +ffin s +thi eves +team mate +pra b +om er +ta pes +bo d +mit su +ste w +e re +p bs +tu sc +lo we +ra de +parliam entary +h m +ed gar +ðŁijĩ ðŁijĩ +to a +a gh +hon i +s late +ge ek +ap t +hard t +ta p +horiz on +grow th +make over +hi l +paper back +id an +reha bil +gi u +possi bilities +let tu +fran co +bo ss +ach er +does nt +mo e +ta ker +huss ain +ml k +di l +th ia +ham a +real ised +raven s +curric ulum +m ith +k night +ted x +r v +isai ah +cumb ria +birth days +f ing +pre z +mu barak +exquis ite +clear ance +y en +par i +ev o +à º +modi fied +app lying +imple ment +disco vering +chap man +indie game +dis k +crowd funding +mach in +li vel +sty led +âĿ Į +ma king +rehear sals +nutr iti +subscri ption +and ro +cre ators +car ries +ky lie +cam den +appren tice +tax pay +c ca +tuesday thoughts +pis sed +er man +dete c +freed om +mer i +.. ! +psal m +sun light +per spec +be ings +book store +rock star +fun ctions +p ence +fav es +z n +obam acare +sp ill +coven try +pi geon +pi vo +ba it +kol kata +av al +don or +wa h +privi leg +tra ditions +rajas than +ten ess +portugue se +yn es +tack les +de fic +tor n +pol ling +thor ne +in a +bened ict +bar ry +cal ories +ver dict +save the +nor ton +off ice +main stream +impro ves +fr on +respon ding +real tor +scotti sh +de clar +r l +shi v +supp lier +re sting +swee ts +qu i +. âĢ¦ +whit ney +startu p +thank you +teach er +h alls +ha ve +hand made +pro ving +quar tet +ro chester +li an +virtu al +mend es +of icial +mid lands +x box +meas uring +o vo +accommod ation +bri des +collegi ate +intellec tual +in car +ni ag +ðŁį · +sf w +coco a +co ats +civil ians +presi dency +mat rix +sweethe art +tri athlon +wag ner +ra dic +plann er +the o +execu tion +k um +the walkingdead +sc ar +ro tation +blo gging +bom b +re son +bb les +st are +assi sted +e do +brand ed +war nings +thor pe +acknow le +satis fied +sho res +ri d +dor a +phys ically +bi gh +appro ves +ha h +ric al +vers atile +pret end +lu m +ab hi +ye e +sp it +ãĢ Į +dj s +ash tra +j t +ven ues +gram mys +cy clo +tr acker +over watch +repl ica +el yn +nr l +lind sey +hom o +ballo ons +kitch en +si s +am os +ende av +ðŁĴ » +a rec +thu g +hoo ked +hr c +new york +bur gh +americ as +patric ia +ug u +ap athy +ha st +psy chi +cor k +petro l +ðŁİ ¬ +ak u +po pping +psycho logical +au x +g ma +cad illac +wa ste +auth ent +bri stol +nam e +que er +to ber +jer ry +com in +ch ant +privileg ed +op ar +lo ser +tex t +mar ker +stri es +equ ally +ak i +christ mas +gare th +ble w +em ma +imag in +se als +che at +conditi oning +j ana +ren s +dar ies +o asis +disc ounts +coun cil +i ka +shir ley +vou cher +al ps +w x +q r +dri ft +attemp ting +ut c +Ø ª +gonzale z +m f +jo ker +paralle l +pa re +aspe cts +proce du +n p +am a +rale igh +bright en +gu ire +radi ation +cre scent +ho b +il le +str and +v ore +n ard +che st +di wali +av atar +al der +d ling +pa thetic +ðŁĴ ĺ +spir it +jor ge +film making +ðŁĻı ðŁĻı +challeng er +b j +down town +ht ml +ade qu +twi sted +in ely +( ' +wra ps +oper ational +y ne +n us +mag net +market place +health ier +snap shot +dam on +inter ven +fe derer +ow ls +biscu its +j p +ro deo +blue berry +lec tion +fron tier +summ ers +re yes +pede strian +go l +caf fe +refur bi +bou lder +me ghan +speci alty +la ss +e i +suspec ts +appro x +rr r +ra th +st im +cru shed +he d +wh un +lo af +cr ore +river a +gene tics +so ck +wa sted +ny pd +answ ering +do ve +bel la +ol in +du n +fi ji +pre tty +spar kle +y un +j d +euro pa +li fts +am ber +mu r +te k +boy d +roy alty +in do +ri b +go tham +ti est +inst alling +ke mp +the photo +cos mic +) )) +whole sale +loy ment +eas y +su ing +sett led +af p +pro ver +suppor tive +re es +ne ath +deli ber +c é +wel come +pic oftheday +new born +pat ty +sun s +si est +fl int +diffe rently +spo ilers +troop er +g ins +cor y +look out +equi pped +ta pe +to by +resear cher +u sh +ke yes +al ma +induc tion +k w +k har +sl ick +bri de +e ur +cra ving +book ings +ch es +tr unk +vern on +sp her +cryst als +rel atively +pom pe +uni ons +val ley +par a +w ant +ok c +de af +ser gio +len non +sh ay +cr a +v at +he e +t we +liqu id +pol y +ðŁİ ģ +b ent +be aring +motor sport +bar be +te sti +han i +fin ancing +astron aut +water colour +ri sh +comic con +gar t +wr ong +ber n +it an +ste pped +fil ters +c low +me x +dem ons +all o +expand ed +comm and +et ers +go ats +si ri +y r +pot tery +mari on +i le +el an +san to +person a +du ke +hom eless +li ghted +wheel er +chang er +cab bage +sur real +ham burg +sma shed +str an +k not +i art +ob i +be dro +di al +th ick +b ingo +fu s +vacu um +con ve +ati ve +accur acy +accoun t +re fer +ri z +spider man +ban a +r ite +u b +ab s +medic al +lin k +si em +> >>> +be tra +g lowing +re actions +pupp et +spa ghetti +ang s +re medi +pray for +roy ce +char lotte +£ ï¸ı +gh et +affe cting +ro de +soci alist +mo ses +az i +o it +re porters +cd t +ap ing +s nat +minim al +wa ist +sie ge +>> >> +ri g +schmid t +h are +ec a +thor n +he mp +es the +cly de +th a +don ut +moham ed +ling erie +le gg +carpen ter +perform ers +de a +imag ined +cur se +la sh +ct r +agu a +ro ar +gr i +ro le +j fk +resur rec +roosevel t +maril yn +sm alle +will is +wa ited +char ities +the res +li k +origin al +car i +c ough +cru ci +la gun +contra st +k ou +arm our +re moving +t ent +maz da +bri ghter +thi ef +cor ner +tequ ila +buzz ing +al bi +p am +az ure +disc oun +pixel art +possi bility +ham ont +tra des +bu da +hi ve +vers y +fin ch +tran spa +em i +terri fying +in qui +g ba +sub stitu +collec ti +plac ing +cin dy +k ann +pa tho +diamon d +mour inho +guine a +anthro po +air s +pu mps +ì ļ +pas o +cur ling +an ita +resi dency +ne wh +jo on +cigare tte +que ue +ex trac +gam es +spl en +ex press +public ly +bon nie +tribun e +ba ek +reason able +c or +timo thy +she eran +Ä ± +f dn +su tton +concentr ation +carav an +x avier +al ger +cy lin +freder ick +ner ve +pe ak +lettu ce +j ail +pre game +kav an +up graded +eco logy +squad ron +gra pes +goo g +pa stry +ðŁĹ £ +ãĥ¼ ãĥ +mil ano +awa z +presen ter +ðŁĮ ¿ +her d +king s +tem plate +fl our +h v +k ley +i ya +spe c +at er +frankfur t +co ch +tex ting +del i +communi st +regi ment +ele anor +anticip ated +ðŁijĮ ðŁı» +thephoto hour +ran o +survi ving +simul ation +daw son +ar in +aqu a +m or +âĢ¦ . +cin o +ira qi +sh az +dun dee +we s +dra u +hann ah +s news +occup ation +ste en +x m +ang les +sett ings +gur u +kno x +or ca +shap ing +w ent +dr illing +zz ie +br i +kis sing +fin d +ma ine +âŃIJï¸ı âŃIJï¸ı +ðŁĮ į +lar ry +bu sted +ta vern +acti vely +- " +replac ing +no d +un lock +. " +âŀ ¤ +affili ate +to w +l n +happy newyear +di f +j m +green wich +contro versy +daw g +con dol +sav annah +compens ation +touch down +te o +amb itious +embro i +convic ted +iart g +bar ack +tr ance +testim ony +au dition +thum b +my ths +be x +que z +orch id +den y +entit led +hoo d +gr ant +in box +blue jays +r illa +smalle st +bur den +in famous +divi ded +boun daries +t ter +el t +wy oming +be verage +me sm +one ws +budd hist +y ana +as sad +is ms +bar rett +predic ted +back to +tw it +e there +cap tains +escap ed +ay o +lam borgh +gard ner +la ps +k al +adverti sement +insec ts +na po +am en +ac y +r and +g k +te h +k athle +tri dge +pan cake +at ro +pyram id +bu la +paral ym +gau ge +en cies +tom y +biscu it +but cher +quali fier +coun ty +ke i +po ols +dar ker +should ers +ðŁĩºðŁĩ¸ ðŁĩºðŁĩ¸ +sp re +( " +writ ers +g m +ðŁİ ĵ +k nit +hu ff +mt b +philli es +o st +den is +g art +licen sed +inter face +ex cel +d well +from the +co fficial +az zi +appear ing +fore st +n ana +ke ith +manufac turers +beck ham +) ? +e se +col ony +delic ate +ut ter +mc in +transpl ant +pre ferred +par d +ari e +hu b +po ds +perspec tives +pic t +del u +app er +be than +p mo +crimin als +femin ism +sh ack +circum stances +fel las +prote sting +wa x +sugge sted +t ator +dre w +om ni +fa ke +kath y +re b +del ine +ber ni +mi sty +ðŁij © +er able +break through +men swear +millenni als +chan yeol +la z +inser t +rep lies +phra se +n x +ihear tawards +audre y +gran ite +rac ec +ori e +ter ra +innov ations +britt any +at eral +pe ar +bio logical +sh ments +institu tion +m sn +frequ ency +d man +neg lec +t f +ste fan +fox news +ty po +comm s +sequ ence +car men +wh ites +econom ist +exe ter +se um +re sorts +cas ually +bun de +divi de +Ø ¹ +ga g +cre ed +reti re +cau cus +rapi ds +wrestle mania +tul sa +sunder land +fundam ent +o di +yam aha +v ary +intri gu +el se +be acon +an gie +tra ded +tran sm +g ents +kn itting +gal ac +ðĿ Ĺ +u to +sea side +hol t +re rs +far go +train ers +mon soon +b ale +sou ght +mad die +h w +co li +fr an +fav s +ðŁĴ Ķ +int ent +r ally +s bs +lemon ade +barack obama +bre ad +stick y +explo sive +chel ten +t j +as soc +ram en +hom ies +v log +mi ster +lor d +âĢįâĻ Ģï¸ı +aly ssa +sketch book +ru mble +cat ch +migr ant +discipl ine +un likely +chronic les +fl ora +sl ams +am id +s boro +coo p +ju mps +tran qu +mel is +sof ia +en ri +gab e +sy ri +nicol as +cha i +w v +be cky +foo ty +ta o +suppo se +ðŁĺįðŁĺį ðŁĺįðŁĺį +plu sh +ri sh +ðŁ¤ ĵ +k ha +satur days +ac cent +he c +lim it +carl ton +wi red +taylor swift +ðŁĺ ij +sq l +har ro +recipi ents +g at +go p +th of +amaz ed +gh an +ðŁıĨ ðŁıĨ +por to +cla re +di stant +na c +ohi o +ðŁĻı ðŁı¼ +mt n +anti bio +dino sa +me sa +par tial +b v +lear nt +lov ato +questi on +ex tract +gossi p +gi bb +niag ara +ðŁij ¨ +displa yed +so oner +ste vie +nug gets +ml n +bro m +tur b +give aways +stu pi +bl ink +c ili +conven ient +mo h +vi ve +f ric +cau se +cham ber +cu les +ne arest +is se +small biz +t j +canadi ans +smar ter +bra sil +ra re +que tte +w ha +cand le +at omic +ðŁijį ðŁijį +warri or +relax ed +stri ps +ne ur +k ka +r fc +jen sen +reco vering +respon ses +sal am +ortho dox +acti ve +ell ers +n it +âŃ IJ +metro politan +centu ries +vi da +gra ding +transpa rent +sim ple +do ts +superint endent +elev ator +autom ated +red skins +ima m +summer time +jona than +ge aring +michel le +confl ic +m ice +to te +publi sh +pa x +) - +na iled +á ´ +tele scope +ser bia +ba b +ape u +st ically +sen ti +r ats +isol ated +grou p +hat red +paranor mal +stan ley +ali on +safe ty +l s +ठ° +nex us +alexand ra +mas ks ++ + +tr on +au k +brother hood +brow se +mix es +sim one +mu sk +appro ve +lo la +ex p +per th +fu turi +un seen +d m +chel se +sc outing +o we +portsm outh +k ram +mi ze +di spen +su p +d lc +adver t +tere sa +is le +cy cle +met all +shi elds +marin ers +ra z +ing en +fun d +an go +jon es +o ka +mad den +broc coli +domin ic +situ ations +mer o +cric ke +puni shment +d b +sha king +ðŁĺ ļ +m q +ari ans +le h +cla w +we ds +d ure +ni el +j elly +gour met +tra ders +le vi +w ages +kne es +wi se +heaven ly +avi d +melo dy +z ack +ban anas +apprentic e +pro p +fun ny +o de +respec ted +me gan +fe wer +dra fted +med it +gra pe +us army +cru sad +vo cali +prepar ations +non sense +us age +th r +ro th +wiz ards +insi de +promo tions +mon a +red sox +si g +eleg ance +ch ia +univer sal +ãĢ į +ra ja +un ga +pol lin +filip ino +ak a +t sun +ik on +bi king +decor ations +z ac +cade ts +hum our +ag m +re ppin +vac cin +elo ve +u w +dia be +galla gher +az er +do l +a while +pro minent +wel sh +t ann +' ) +bi en +wa g +in al +c wc +wic ket +ur st +q anon +x e +out door +dun n +star r +co logy +ric ky +u efa +reb ounds +s music +inf ant +ðŁĻ ĭ +so p +u mber +hand ing +beg in +sor ting +ha sh +sp ati +re k +buda pest +black hawks +dele te +ro m +can did +auth ori +de bris +spe cul +inter section +marri ott +im ran +ðŁĺģ ðŁĺģ +cru ises +ram sey +rafa el +aware ness +vas cular +beyon cé +ru g +ðŁĺ Į +festi v +ar am +s able +bas il +p ill +flo oring +un beaten +implic ations +u f +w ound +for ge +poin ting +po ts +popular ity +ðŁijı ðŁı» +mani pul +s lots +deb ates +abs ence +ver mont +never forget +wri st +gl oria +ren ce +hu sk +mel ting +ðŁİ Ł +br aces +tim ely +transform ing +am ps +ma k +po e +ah an +gener ally +nd p +ale ppo +unic ef +pro fs +nor d +ma sk +jackson ville +v v +sh ells +bloom ing +oper ators +char coal +ne ville +ma gi +chi p +sam a +ir an +re forms +accu mul +ru e +æ ľ +web sites +ga on +devast ating +sto s +glaci er +ra pp +chipot le +pr a +or ous +rom ney +seas on +decor ative +c isco +dit ch +compla in +ll o +assu me +ðŁĺĤðŁĺĤ ðŁĺĤðŁĺĤðŁĺĤ +n els +cent ric +ft w +car rots +tat a +can ter +per ience +li ers +demo s +bl unt +oper ate +reserv ations +le ah +sub stance +di son +an te +elec tion +v ue +squ are +non profit +ca a +f su +y am +ãĤ ¤ +v ladi +comple tes +mar i +philli p +ne ill +er as +ka it +men do +mahar ashtra +g p +dan e +provi dence +ther apeu +juven ile +me mo +in corpor +aa aa +seven teen +teen ager +à £ +or ns +wi de +cu teness +tw d +ff les +bar a +com edy +over time +y az +bar on +unemp loyment +ðŁij ĭ +exter ior +den se +cent res +match up +history month +artif icial +qu it +e sk +war n +cr itic +j af +ðŁĵ ² +inform ative +fu els +recy cle +nam ing +stri pe +sol ic +mole cular +dee pi +con vo +s sel +na e +de scent +ti z +accoun tability +ter ry +r ito +sl ay +em o +dem ol +sens ation +co v +tor e +round table +y ol +excu ses +ॠį +tur quo +hh hh +pod casts +cele b +me ssi +li o +man n +contribu ted +u z +gener ator +ele ts +veg gie +indu l +en suring +detro it +pun jab +tran spor +instru ction +ad d +por cel +pan eli +cir cles +persi st +clay ton +sp n +dog softwitter +is nt +sp r +retail ers +p w +hun gar +el ena +mon aster +gu atem +je ssie +an z +ra shi +fle e +car ving +fau x +l al +hen ri +d jo +du ll +s ana +lar a +glo be +cri mson +com pass +pau se +na b +lion el +ba ths +u fo +invent ory +sin gh +sat an +ðŁĩ ¸ +ce ments +in form +gener ated +bi den +av g +tas ks +de er +sa u +ja iled +pa stel +sc c +na il +steel e +per is +lamborgh ini +pur sue +mar gin +u ch +bo sch +dra in +cl ara +bo m +lat ino +web ster +rose mary +r ha +s oun +billion aire +not ch +percent age +con or +' " +hom es +earth day +h ort +big gest +di sin +wal ton +edit ors +im ma +om ar +equi valent +pharmac eu +ah med +cam eo +han ni +under rated +ge ment +micro bi +v oo +honor able +obe sity +âļ ¡ï¸ı +limer ick +invol vement +st agram +boule vard +bur g +blackand white +liber ation +fi ve +inter im +sm m +rival ry +cap abilities +stat ements +thu mb +ve d +sw ans +bar ber +e que +seren a +hel m +noo dle +sam pling +n awaz +sing le +thunder storms +sh on +in ev +ë ¯ +to pp +orch ard +bi an +ðŁĺ Ķ +door step +salv ation +marke ting +r ons +cle mson +ra vi +in take +stand with +sin a +ha iku +ple y +elector al +ph illy +la ys +electr ic +cap turing +u pp +er gy +believ ing +cul tures +es day +inva sive +ed ed +spee ch +end ur +viet nam +boy cott +pe de +deli ver +ðŁĴĸ ðŁĴĸ +mer chant +st ir +den ies +poc kets +o ti +cu ddle +ro land +mm ed +den ed +lear ners +hoo p +sour cing +h acked +di m +environ ments +ben son +jud icial +wor cester +pear ls +govern ments +arri vals +cor ners +tun ing +la bour +y m +or dering +le wi +i fe +hygi ene +thou ghtful +indone sian +campaig ning +princi ple +assau l +ru bb +at v +wil ly +en tre +il i +ph on +du ties +âĻ¥ âĻ¥ +sn akes +lo op +am ar +conver tible +bon ding +ment oring +max well +ethere um +destro ying +ax is +ca iro +fin nish +sho ck +ðŁĺ IJ +cal eb +com a +pe dal +co re +contin ent +el son +temp o +helsin ki +ac p +tack ling +st ated +bl a +dou b +sma shing +a ja +camer on +disru ption +warm th +being salmankhan +bullet in +o de +syrac use +ar an +mc gregor +bul k +an ton +confir mation +sp ine +im ran +instru c +jac ks +chi o +pal m +str e +embarra ssing +un t +elimin ate +to ss +c ise +a ws +oni sts +sh inee +jo s +ho se +li vely +opp onents +mo vements +recogni zing +sandwich es +sh akes +exerc ises +se at +profe ssion +merry christmas +lu gg +adopt dont +mar vin +byr ne +un le +he t +ku wait +rah man +aspe ct +humb led +gen es +f and +long time +) ; +cam pu +an gus +ðŁijį ðŁı¼ +q uran +sle eves +s lic +¸ ë +twel ve +your e +i ke +go gh +b st +dic tionary +reflec ting +to on +yar n +em bed +ðŁı ´ +re serves +floo ded +ver iz +du sk +estab lish +pro li +au d +ritu al +or bit +declar ation +recor dings +cam o +cas sette +good luck +cu tter +bo p +b ho +che ating +paci fic +ma res +tim er +col t +tr ous +tomor row +han sen +ci e +w ang +ban i +circu lar +ac ute +far mer +co ys +p se +ir ving +w j +haw kins +b ison +ur day +cru ising +o te +k ath +whi stle +your selves +ant is +sla sh +thorough ly +ke sh +ser ie +ex em +en ig +guil d +sh red +ho gan +ap o +ä ¸ +pu zz +ne tball +au ssi +panor ama +ws j +av is +ar ming +hum ph +brow ser +cri es +fo ggy +mat te +ðŁĮ » +it er +tal lest +by ron +cap tiv +je su +any ways +flag ship +p ton +we y +fay ette +financi al +f oul +solom on +jenni fer +cucu mber +ar gue +tex tile +wrest ler +john ston +pa stor +ðŁĺŃðŁĺŃ ðŁĺŃðŁĺŃ +cac tus +edi ble +re served +ric hie +met res +ingredi ent +h ella +un to +ch ol +cele bs +po ets +gra ham +hay den +coinci dence +b aw +communic ate +flet cher +/ - +tole do +ecu ador +coun sel +s laughter +line ar +at p +os u +jo el +ev ed +conqu er +ru stic +plic ity +recogn ise +room mate +cr acked +jas per +ph er +ðŁĮ º +wo ven +mo ist +ff c +ste ering +ni sh +stand ings +frequ ent +ar di +haz el +as msg +bau m +d art +si dd +nat h +ch ero +card board +c ss +n sfw +pa ir +ðŁĺį ðŁĺĺ +occur red +homeless ness +mal one +ph e +xi a +pad dy +decl are +theat re +b f +per sian +ta d +ax e +susp icious +lam b +mu cho +sen ior +st as +k ite +st ing +gra d +k af +wat ering +Ø ¯ +spi ral +th ms +educ ator +jer ome +of c +clo ck +su l +pe mb +.... ..... +park way +de aux +restric tions +m ons +need le +e j +le agues +water melon +am an +pl enary +max im +w ab +coming soon +bry ce +vi gil +super market +fortun ate +turquo ise +presi dent +li v +inter ns +feel in +fix tures +stun t +st aged +premi eres +lo k +prac titi +shor tage +log ne +ve c +con cor +roc ke +li g +com posed +syn thetic +di p +cam ila +ch is +j ou +su san +eye brows +supp lement +satis faction +moham mad +ti bet +house of +pu n +as sam +shado whun +psy ched +se duc +mand atory +her bert +sc allo +stream ers +proto col +block buster +produc es +sch nei +lau rel +tri be +time hop +pl a +mod elling +tv time +mtv stars +wi dow +me tric +ch am +con do +flow ering +ale c +d ms +inten sity + ¨ +mccar tney +islam abad +k b +f fi +ph al +anal og +f ond +h acks +positi vity +treat y +sub marine +conne ct +sel en +categor ies +cu b +organi ze +si k +quote oftheday +remin ding +am or +loc king +ðŁijı ðŁı¼ +comp ound +et te +b out +rec ur +fe rence +mi zz +tren d +hip ster +for tress +forth coming +preli min +o dyssey +ang p +del ici +even ings +ðŁĶ ¹ +i q +d w +da ir +kathr yn +christian ity +moon light +ha b +wh oo +f bf +se th +genu inely +pa x +char ity +deplo yed +b nb +bu cs +ju dg +con ge +plant ation +im press +car a +sc lub +sco py +land ers +compla ints +b ama +re build +x y +real ism +sh our +le in +brac elets +mer a +assas sin +an chor +ðŁijĮ ðŁı¼ +lin en +con fron +chronic le +comm ent +cat alog +il les +gor ge +me try +jung kook +love my +sent in +se em +fit ness +alli ed +ts man +digital transformation +pr an +lo ft +min ton +alden richards +en vel +cher ish +certain ty +zz z +rhin o +per kins +en rich +cape town +ome ter +sec tions +ske leton +def enders +ðŁĺ Ŀ +pen c +bri t +ja h +capital ism +ðŁ¥ ĩ +baz aar +re me +ex t +kk k +conver t +stor my +b ye +kar an +chry sler +ad os +pre ssed +syn c +ation day +dang er +bad ges +refu ses +em powering +ly m +ex ports +adoptdont shop +ðŁĩ ¯ +th c +awa ited +focu ses +fin ed +o at +haha hah +âģ © +n family +fi ona +luck ily +thr illing +ty ping +out break +di es +he u +craw l +ne sses +o ath +scri pts +gee ks +ðŁIJ Ŀ +p b +mathemat ics +al is +________ ________ +gymna stics +acti vism +recommend ation +gre n +wa in +cour ty +n apol +cau li +hor nets +g als +jo ckey +dir ty +at ar +enor mous +pe st +greg ation +an os +ii ii +def ends +black historymonth +at x +mb c +lugg age +wit ch +co b +la sts +cu m +gg g +ba thing +n ar +ce bu +ðŁį ĥ +navig ation +min e +re jo +ðŁİ Ģ +gif tide +re ta +use less +pu ll +defic it +al lu +ati me +it v +tr illion +pu e +ac ies +proce dure +l ori +jen ny +c ad +ul ously +dr ac +promo tes +ing the +can u +woo hoo +na omi +zar dari +ts u +be ir +sd g +le ver +we ber +ab ud +lun d +crow ded +deplo yment +ter rain +ken ny +ho f +witne ssed +lo ch +j k +bul ly +w ren +poe try +do ff +ww i +mo red +din i +cul ture +promp t + ¥ +maur ice +to pps +r m +cor respon +ab out +jewel s +gi br +eag le +ðŁĺĺ ðŁĺĺðŁĺĺ +l ending +sou ven +ç Ķ +contemporary art +establi shment +j ong +âĢ¦ " +gat or +patri otic +mc coy +v ape +human e +feli z +coach ella +re posting +ste als +fu ller +n ering +at ra +( - +bla ke +he ather +wor ms +discipl inary +rede mption +y ard +am in +" @_ +d nc +t ds +k appa +ne wark +comm its +spe ars +j ams +t and +msn bc +inter medi +aim ed +at ic +teen th +observ ation +kash mir +kavan augh +ou l +san francisco +re u +bel ated +cho w +pass word +st ills +deta ined +sar i +day ton +dar ren +itali an +ar th +amu sic +ar bit +w m +v m +he m +dou g +my r +a sho +pre v +vin d +bra h +sta g +ภµ +pre views +gu k +con taining +leon ardo +sad dle +ru shing +st av +lon gh +gam bling +ve gas +reserv ation +end ale +bal a +fl a +vari ant +he dge +bulgar ia +nat ali +we aver +sol st +encoura ged +ap c +as parag +ne st +cycli sts +fe l +ìĬ ¤ +overwhel ming +pey ton +j it +a post +mb le +ble eding +neighbour hood +a very +expre ssions +mac donald +gi gs +mon ds +illu sion +n ct +cam ero +over head +my th +ol y +vi o +et v +lau rie +unve iling +pri or +con n +iron man +di ff +day in +crit ici +con go +re vision +wal e +direc tor +p ines +black pink +gar ner +cur ated +manit oba +h ac +common ly +bar ton +.... # +mor tality +live smatter +philos op +shor ter +con vince +fre ak +vend ors +insi ghtful +el ly +sens ors +e led +s berg +weight loss +u kip +sp ur +priv ate +qu a +ss c +, ... +supervis or +advis er +amaz ingly +less er +at es +mah on +oooo oo +sar as +pmo india +waff le +un ders +toler ance +sculp tures +her sh +kno cking +smo ke +cathol ic +gri m +tra veled +fli p +ge off +dinosa urs +sle pt +scar let +ok i +compla int +ob sc +nam i +la g +cross fit +u fc +mc cain +refe ree +sad ness +pen ny +li eu +mo de +ki er +vol s +w is +el on +she a +ba o +son ia +cla ire +em manuel +moist ure +di gest +vi ii +t eller +ch on +access ory +night club +foss il +aw an +hu sky +ab original +brand on +ffici ent +cou gars +ste d +ad mitted +igno red +content marketing +ag as +v ase +execu ted +negoti ations +she ad +n and +tab lets +go th +ts al +d fw +on ep +protec tor +sp ho +gaz ette +andre as +ss er +comp ilation +ha v +contain ers +bro ker +soc al +porcel ain +hy uk +air ing +ðŁĴ ° +publi sher +scen ario +spart ans +re viewing +itu des +ed el +pear son +ba sh +mau i +a ad +ðŁĮ Ĭ +li u +ul ate +program mes +fav our +web design +real ty +motiv ational +cro sses +' ... +bus ch +adjust able +ar jun +mist ak +dimen sion +pi stol +weigh s +en y +unve il +indy car +gor don +f ade +fran ken +qual ities +bet t +loc ate +ker r +sp c +confu sion +ne e +luck y +bas es +dep ends +fire fighter +ol a +re t +mar oon +ðŁĶ Ĭ +w am +defin ing +whe at +bi l +é s +b hai +psy ch +ta u +ic ans +thi k +ob ile +inspec tor +ìĨ Įë +ill on +go s +ev angel +fa i +si st +voc ation +bur ge +chi stan +renew ed +enthusi asm +en ting +ag ri +ike a +m sc +aero space +sens iti +memo ir +hosp ice +co caine +der ry +mechan ics +Ħ ภ+tin o +reduc es +collec tors +in justice +supp re +v ana +ab un +nap a +su sa +os lo +e ff +en core +lic ence +ched dar +z al +moun t +ðŁĴ IJ +threat ens +!! " +archi e +fu tsal +scu ba +jo s +gn on +se xi +s official +compar ing +domin ant +tof theday +fa it +propos als +gi ft +y as +cn c +l r +ha b +reser voir +beli efs +gener al +mar ti +t d +est e +ì ł +wi l +ðŁij ¯ +ðŁĶ « +sp x +et work +excer pt +e instein +hir o +sil hou +team ed +per ception +corri dor +mental health +hin ts +ben ny +induc ted +sw x +wi desp +spe ak +cher yl +dru g +ðŁĺ ķ +h f +asparag us +myster ies +fitz gerald +off er +therap ist +care er +dam aging +ts d +per u +wei bo +y ay +phoeni x +disc re +mac book +bar ker +stig ma +sp read +roc kies +kang ar +bri dg +pa i +bi shop +ta iled +capsu le +ðŁĴ ĵ +ge of +roy ale +short listed +o ste +ash amed +ch app +key e +cl a +screen shot +austri an +nati ve +en ight +juli et +michel e +ðŁĮ ´ +travel ers +pi l +football er +win chester +ðŁĻ Ħ +azer bai +gold eng +organis ations +interpre tation +predat or +ofthe week +lo gan +pok é +mari e +cal la +t nt +cin de +ge tic +fit fam +gra v +ow ens +ðŁĮ ± +shoot out +sal is +commissi ons +co he +p tic +ni xon +hi a +amb ition +mar ine +cruel ty +t k +cru de +sal ty +jim a +mon go +ir ony +on wards +arre sts +strang ers +ig er +cycli st +ra g +exten ds +tra dio +bour g +mo i +el la +e able +lex us +au l +der a +histor ian +mor ton +ti ff +man ner +ko t +d k +po inted +mar qu +a an +en ey +du blin +on poli +em ili +secre t +fl o +âļ ¡ +ba j +ste ep +accompan ied +rum ours +dev i +purch asing +fi g +pu b +sch oo +autonom ous +go alie +x ia +autom atically +re vers +ter o +fu ku +titan ic +shoo k +sand als +see kers +exc av +nor dic +bigo live +ba ke +r att +z ak +ne p +ðŁĺ ¤ +cand y +billi ons +book worm +pp et +à ³ +sur faces +sc ars +phil ip +do gg +ci gars +co te +transl ated +cur ator +sin dh +han gover +bre wer +on es +el ton +ðŁĴª ðŁı¼ +mar cu +elli ot +righ te +di oce +ru ss +rail ways +grand son +as cen +apo logy +awa it +mob ili +re spir +parti san +oli vi +stri ke +yo o +white house +expre ssed +pu ps +bed ford +cul tur +fro gs +fly ing +cav ali +c ds +fri ger +street photography +re solve +tali ban +kan g +cru shing +ju m +ðŁĺ Ĵ +william son +tan g +cur ly +t man +veter an +fa ire +artificial intelligence +un anim +pre n +back drop +fr ances +oc cer +doro thy +work ing +ar thr +conver ted +day light +serv ant +pad dle +compla ining +thir ty +nad al +ak u +ibra him +ad dressed +p iss +green house +batt alion +si mulator +out lets +embroi dery +ðŁĵ ± +fis cal +ger ard +sas sy +ðŁİī ðŁİīðŁİī +vent ures +mer it +public ity +ðŁij Ī +sophistic ated +c tu +conven tional +condol ences +isra el +tra dition +ar an +te ss +gla d +ðŁĺĬ ðŁĺĬ +correc tion +ge on +am d +or ship +be ast +ch ment +ì ŀ +nic o +wk nd +wel s +cushi on +beli e +vo c +idio ts +under neath +pu ma +corn ell +en ation +lu l +swa ch +ab ig +u rer +mi e +form erly +ca f +er nal +chor us +juli us +sen ator +âľ į +wh ir +salv ador +ph d +uni fied +boo ster +graph ical +w rec +son ny +mi z +dere rs +s all +ven s +tusc any +wi d +y ong +kur ds +w az +trol ls +mac ro +cat urday +pre ssing +sa sha +cent ennial +gu sts +em c +be fore +den ise +cu st +ðŁĵ ¢ +lo oo +base l +eng land +y olo +ar du +manife sto +do ha +ì ľ +kni ves +bourne mouth +bi bl +bar b +al icia +Ø © +com er +cycl one +g it +ane ws +character i +vent ura +in tra +sf giants +hu t +be a +dar win +ell er +al v +re ese +bl y +kar an +conclu sion +man ny +fla kes +unite blue +nad u +co pp +ed ges +lanca shire +i als +o tta +philipp e +l ent +che e +ment ors +festi val +an ism +compli mentary +r j +pu g +d ine +we i +cli ffs +sar my +ti veness +treas ury +il and +after math +rabb i +ou n +bou quet +herit age +zi on +sur render +shen an +in ks +kar l +gh ty +pol icing +exam ination +ce y +per su +measure ment +hydro gen +lu han +âłĢâłĢ âłĢâłĢ +war i +о Ð +j y +fow ler +mis h +al fre +âĺ ij +bb naija +cat alogue +recogn ised +sa ver +hu skies +col in +mun do +si va +p ng +discoun ted +man utd +fre sno +de vin +prelimin ary +tro phies +pla stics +du g +pro cu +indi go +g ard +dy lan +pit ches +ground breaking +in son +bl ac +an thology +f h +expl ic +r ard +admi ral +so chi +la shes +splen did +en vy +ad v +sex y +festiv ities +stic king +bi b +thr ill +op p +ari el +botan ical +endur ance +fe males +br icks +vat ican +black pool +ber mu +br ough +roll er +bi d +sue de +sloven ia +mm ing +ml b +med alist +di ans +rehabil itation +ne on +s go +li thu +ram os +z ed +pi anist +inten sive +broad band +stu dy +peter sburg +lu ca +ah hhh +phys ician +dill on +tele com +gri ef +mu n +ac ro +si ded +s ly +blo ws +classic cars +tri um +ar gy +? : +h ri +marsh mal +âĢ ĵ +to pping +war saw +tran sc +preserv ation +b av +re friger +experim ents +ä º +gl it +sli ga +g age +fac tor +flav ours +br ony +sp o +cook book +carri age +aw ay +ny fw +on ian +w g +simp sons +ro lex +ðŁı ¿ +cro sby +ãħ ¤ +cre di +syn dic +pu bs +ali fe +poor ly +mac ed +ðŁĺ ŀ +behin dthe +w enger +n ats +ðŁİ Ł +rubb ish +procedu res +typho on +opho bia +er do +fu el +vi era +bu mps +millenni um +new zealand +lec tures +it on +mil ky +respon ded +ê ° +landsc ape +.. @ +bo ther +âĸ ¶ +z hang +huawe i +tu ition +s worn +in u +y or +pa olo +au ditions +ab il +malay sian +ho ps +fe athers +mp le +au ts +ã o +boun ty +ic he +ì ĺ +sh q +pin ot +ge ars +disapp ear +video games +t na +alzheim er +ðŁĮ ŀ +a ji +under wear +swit ching +sign age +o scar +ec on +dro w +cl int +pl ated +gun dy +emb lem +ho es +ici st +nel ly +juni or +road show +miner als +at le +alexand ria +ac claimed +v ell +shi va +ad he +en ne +amne sty +h ounds +councill or +ðŁĴ ¦ +aes the +part nering +influ enced +mag no +fl are +extin ction +civil ian +maje sty +va il +law makers +rac ks +mc c +ori an +sp ices +er rors +may er +co ca +pa i +s ooooo +reti ring +ba thro +ðŁĻĮ ðŁĻĮ +âĸ ª +su f +endor sement +buil ding +broo ch +pal la +arvin d +ag ent +kar ate +r hi +c tv +ta ine +um m +ba x +reig ns +uni of +enterpri ses +adel e +fla ke +at tire +bru ce +ba hamas +gra vy +sa in +che ek +tri vi +lo v +e en +bb lo +lady gaga +itt a +. "- +du stin +observ atory +eigh th +bloom berg +kh s +f cc +gi st +commemor ate +ve er +sexu ality +ed c +nic ole +vac ancy +u ser +son a +:' ( +dipl oma +t end +up grades +Å Ł +jura ssic +cardi ac +dr s +widesp read +à ł +dail ies +vend or +sim plicity +wi der +len ses +supp lements +de pos +ob served +vin es +parti ally +renew al +collabor ate +ali g +fin ity +ph u +zz y +pe tit +ðŁĵ ħ +z in +i gu +sm ack +fall on +ðŁĵ £ +back wards +comp onent +o so +compati ble +bin ding +zur ich +thom e +w ounds +ly ric +fresh men +sne aky +fi bro +di et +emplo yer +in sect +h ated +sch er +raz or +n sw +boo ker +califor ni +av fc + ° +preten ding +pep si +al is +un titled +k art +grand parents +e the +o ck +lux emb +visu als +small business +abdul lah +min ho +su baru +h ra +reve aling +heart breaking +clar ity +am g +sl r +** ** +âŀ ĸ +recor d +ici ary +min ded +ye h +exce ssive +knu ck +icec ream +tru th +ev ic +ta stic +ant arc +ren dering +, , +mit t +loren zo +st patrick +bound ary +zi g +vo cab +osa ka +fur n +tu n +gu l +s ounding +blo gger +utter ly +g af +adv ancing +l cd +mar gin +lifel ong +solst ice +sh ra +wa its +ple ar +bre ach +en ligh +ad er +itt le +c ation +ho on +stu died +?? ??? +k ash +ev angeli +ps l +wei ghts +met als +ty res +tur no +wi e +car b +g ale +se al +sun ite +am ic +patter son +á n +eu ph +up stairs +quali fiers +khali fa +apple music +ìĨĮë ħ +vau ghan +al ter +cru iser +mu a +t ana +kat rina +id ols +spo iled +secre tly +fi bre +part nered +um es +gi ov +com et +screenshot saturday +k eller +fil tr +fe t +con way +pe u +bad minton +gi d +m ound +don key +bu ff +lea ther +lar gely +bro ch +int ments +am use +r k +sto ve +impac ted +con t +cr acks +prison er +bar i +contrac tor +ori oles +domin ate +pol ar +am elia +dr c +ðŁijĮ ðŁijĮ +vi st +su arez +injec tion +blo oms +ðŁļ¨ ðŁļ¨ +sti ff +pay pal +sno wing +thur sdays +goo se +we dge +educ ated +weak ness +de cker +abud ha +bree zy +Û Į +hope ful +o bi +rai der +gh am +de u +se ve +par tly +fu t +infu sed +mer ri +than e +some time +hu e +me in +cre dit +sli ding +ran de +cher ry +dead pool +sh ol +ar am +under wood +sky e +distur bing +m nt +poli shed +guardi ans +ha dn +pic asso +ari us +ak shay +ir ri +j h +happ en +la kh +dal ton +at the +s well +mar sha +re h +cour s +j kt +top us +serv ice +r ink +hack ers +dono van +hor o +tc m +may hem +cha se +dev ops +ken sing +sc up +sh ere +quali fication +c live +ton g +n ancy +mar is +der dale +ber man +cinde rella +jol ly +ci c +loo t +collecti bles +hom icide +g ge +epide mic +su ites +mu ddy +gi mme +e rec +- * +tal la +lis le +embro ide +ðŁĩ© ðŁĩª +veriz on +ve ctor +be anie +arti san +ga in +flo res +vi gil +u so +ðŁĻı ðŁı½ +grin ding +gh er +air ports +respon sive +shaf t +can cel +ceremon ies +e me +at ari +bru shes +eag er +bo hemi +children s +yan kee +ma a +suspen se +mor an +mac ar +sun flower +cre w +vo id +ke ar +fashi oned +jen nings +sunday funday +sub missions +me ad +her man +wa i +crit ically +le um +baek hyun +for cing +co bra +ãģ ® +acqu ire +al k +ge ology +pri mar +import antly +ire z +bunde sliga +curi osity +sen a +stric t +con soli +win ters +ven om +chelten ham +ðŁį º +cen a +t at +ba in +glo ver +under cover +as ses +car n +memorial day +am eli +i rene +ch on +syn thesis +spe edy +mitsu bi +sla yer +compos ite +under stands +pe w +inter rup +hen ri +mor row +an om +thof july +g lee +thre e +ðŁĺ ® +and hi +ch att +renew ables +ye s +trans fers +!!!! !!!! +bab u +du ter +lo ops +pe ers +o ilers +pau lo +ic ation +h mu +war a +mer cer +hom eland +fu ji +ale y +year book +re m +re en +ab sur +bo is +] : +caes ar +shot gun +kur dish +o ren +ra e +anci es +ty pic +f h +def ault +re plic +lu k +trans actions +r ys +infan try +ðŁį ¾ +cho w +chick ens +ba gh +wy att +ay e +gg i +bre ws +ed itions +mi ra +commen cement +pre su +peris cope +ic hi +guatem ala +zam bia +pain ts +wit ches +wan i +un dere +cro y +vo ws +us mc +hear ted +theat res +shu ffle +le vel +mul tic +squee ze +fer n +app et +post al +mal t +on board +ld nt +co o +s sc +k ac +ðŁĺ ĩ +sc rap +mar cos +deal ers +ann u +mill er +co ve +ul ary +vladi mir +be ef +th ur +pick led +se same +bengal uru +mo tt +kathle en +hi st +no tor +dr ank +du chess +snow fall +e ff +tin y +j n +sy our +speci alists +scot us +bay lor +eve rest +mali bu +pre m +harm ful +l ali +b ates +g ye +differen ti +and ra +geome try +el over +black out +== == +ko ta +inter act +asi an +la yo +samu rai +fi del +exhau sted +gla di +pd t +spher ic +anti qu +guit ar +stu ri +ho pper +ang le +f ills +sla p +mi th +rod ney +ong i +in som +pre venting +cassi dy +ap ho +ore gon +lo in +ham mond +contribu ting +f n +gar ri +ori on +comp elling +escap ing +aim ing +plu mb +bi stro +be asts +concer ning +bo e +do pp +shop local +stumb led +âĤ ¹ +naz is +âĢįâĻĤ ï¸ı +gest ure +war ts +us open +hi ggins +char li +hang s +bom bers +° : +fe eds +c ch +st il +nic ola +ðŁĵ º +clam ation +tro pic +af ro +ou k +expen ses +der rick +al ine +fa w +reg ard +im er +sat in +thi um +ry der +pear l +te ss +mm mmm +sen ses +ðŁĩ ¹ +positi ve +exhau st +occu r +nor ris +lil ly +is les +direc ting +yo fficial +count less +sam ar +on stage +flo ck +mir rors +arch er +mo i +k d +vi v +in os +si kh +le i +sen sory +br its +kno x +chest nut +op y +coli seum +z af +di vin +adap ter +:) )) +tem ple +ku n +hel mets +t df +gu ide +m old +o ids +lu ther +he is +monaster y +sp ree +k lu +brit ney +jagu ars +gre ats +c cc +ky rie +machin ery +cric ket +re ro +ab o +aspir ing +semi finals +ale ss +sig natures +var d +me th +her bal +hol den +king dom +ap or +reg gie +ore o +palestin ians +em mys +sec tional +ro i +ney mar +qu el +cu ll +l ka +haz el +estim ate +ul ties +go w +be a +purch ases +bel ts +protec ts +m é +gue ssing +bb o +clau dia +fr acking +jon ny +el k +cel tic +al mighty +ra je +courty ard +ig i +can es +ðŁĴª ðŁı» +bank rup +le thal +âľĮ ï¸ı +graphic design +vad er +penc ils +rough ly +dan te +m fg +const ell +cam el +j b +bloss oms +en to +balo chistan +cine mato +ill ard +jer sey +con sent +dent ed +con templ +sch er +hol i +lou gh +st our +a yo +begin ners +cur b +v hs +a jax +du ff +av eng +dom est +commit ting +ai red +cha p +hedge hog +disappo inting +freel ance +in land +char ms +ðŁĺį âĿ¤ï¸ı +ai sh +m x +buck le +ti dal +per mit +bo ating +ra cha +kend rick +b ello +b hi +ple a +estim ates +l b +apo logies +jay a +bb l +ast oni +inter state +main taining +el bow +mu p +ep it +ðŁĺ ¡ +viol ations +def end +be h +sl c +am ir +pur i +ti um +fi fa +blur ry +scri m +ðŁĻı ðŁı¾ +ma ple +rel atives +âĺ Ŀ +cho c +con nor +⾨ ⾨ +whi sp +list ings +ma ze +than king +ri dd +grass roots +shi fting +desper ately +gor illa +den i +ju les +stra th +g ley +ja in +bu ick +t anner +ðŁĴ Ŀ +ga e +pri m +it ors +n ano +separ ation +armen ia +bor deaux +ðŁ ħ +pj net +bu rial +e bon +glo ss +re new +gri er +spe eds +comic books +sym boli +pur poses +ãħł ãħł +spati al +no table +ci on +n ps +ho ffman +nor man +rt g +du sty +situ ated +tr an +k fc +em en +nic kel +hast ings +sett ling +gr it +l ena +w aw +art s +gu m +ca regi +le wis +sapp hire +rememb er +embed ded +t lc +bl at +serge ant +el sa +boot camp +bow man +photo graphic +pill ars +direction ers +classi fied +no is +ve er +barre ls +wh oop +ðŁĺ± ðŁĺ± +fe male +petro leum +medi a +e fc +poké mon +ठķ +enthusi astic +var un +pro files +pedi atric +acci dents +con rad +jan g +jo jo +ac or +ob server +l f +live stock +for gi +fo s +el m +an and +go e +c ere +avoi ding +gri t +om an +thank fully +scat tered +nick y +cylin der +chees y +di ver +mahe sh +cav es +ear liest +qu inte +subjec ts +b end +gul f +vocali st +glu e +pat ches +un stopp +sny der +demonstr ating +pi o +hor ns +wic kets +and the +r ama +yo on +stra ight +bed time +or ang +bul lets +sa urus +min ers +inci dents +! ... +ðŁİ ¸ +ag ers +hand les +stat es +in ity +d ons +incredi ble +emin em +avi v +ru dy +moz art +folk lore +appli ances +mt l +fre y +di as +hu a +page ant +stri ve +im prison +bul lish +r ana +al erts +bb mas +hy per +derby shire +re cre +re dd +debor ah +cosmo s +law son +mel anie +psy cho +ho or +doo dles +sni per +shad y +man tle +canadi an +new year +inter actions +separ ated +cor ds +spiritu ality +ap u +it o +p ct +pel osi +rebel lion +se iz +wor cester +sec tors +ul i +san ta +Ð µ +ðŁĩªðŁĩ ¸ +bi ased +class ical +gam ma +dee plear +emer ge +back er +sur ance +hand crafted +ðŁİ ¥ +franc is +mill an +ic i +cro wn +wo w +stri ped +un fair +relax ation +³ ï¸ı +embrac ing +she alth +pale o +martin i +dist illery +wr ink +or k +na th +hay ley +cour thouse +si ber +sa di +quiet ly +mel t +m sm +me h +smart phones +rel ent +pp ing +war wick +co logne +gli a +cot ton +pro g +lon e +ip sw +star ters +expan ds +u mp +su ed +ski pper +infe ctions +ing le +à ¡ +cler k +demonstr ate +ac ar +ðŁĺĤðŁĺĤ ðŁĺĤ +ti bet +bun s +alo m +demol ition +ssi a +g st +[ ] +so ar +âĺ Ģ +ðŁĺ ª +ðŁĵ Ĭ +dee pest +beyon d +are t +att ends +activ ated +di mit +âļª ï¸ı +high lighted +magaz ines +rum or +az za +steph ens +dol ph +sho ckey +mat s +we av +mel an +serv ers +tra um +ku sh +æ Ĺ +bab ys +pa z +a al +la use +break ers +canter bury +ul ture +mi ri +euro s +tane ous +impre ssions +du tch +il d +gh i +pur due +adequ ate +l p +sy ner +ang ler +du rable +gal ore +ro wn +mg mt +ðŁĵ Į +lu cia +âĺij ï¸ı +zay n +bor row +. ( +north umber +cru sh +eng a +su sh +extra vag +t out +ma hal +ali stic +ther mo +gall eries +es se +chi bi +attrac tions +lex ington +legislat ure +docu mented +resi den +brow nies +w f +st ool +plan ets +sho ppers +conduc tor +ms p +tr icky +fru ity +end ra +feel the +whi pped +hair style +re fer +oo k +oc topus +audi ences +ku mar +after no +op tim +c fl +ni p +gen i +alpha bet +ann ab +lam in +accep ts +l ng +ðŁĺ « +t ine +ac om +cheer leaders +t k +gr on +v g +k ung +ja x +dha bi +r ss +mack enzie +beir ut +clean up +gy psy +st ell +bur ger +hurric anes +educ ation +st ina +âĻ¡ âĻ¡ +unfortun ate +jere mi +bad ger +at ers +: âĢ¦ +ter ra +subli me +stu d +y mca +mr u +duter te +bren nan +bul b +mel o +yl on +hack er +c red +gu d +as an +pad illa +embroide red +vietnam ese +pione ers +projec tion +re boot +id c +an ey +pri mer +suff ers +win ding +p on +sto day +mor n +u ch +all in +adid as +eliza beth +tu ck +o graphy +ðŁļ Ģ +be g +os borne +ghet to +r h +cn n +ir ma +ma kin +cab les +mur ders +oc ks +inst a +al as +si k +cu ff +la re +foo dies +o vic +at om +geome tric +em pathy +ภµ +cent enary +newsp apers +administr ative +ðŁİ Ĭ +sti ve +contrac tors +le tt +tas mania +awesom eness +den sity +ve en +prince ton +frequ ently +re ject +gh i +modu lar +ceram ics +sh ag +ki wi +can vas +sweat shirt +an j +ti mm +napol i +il er +appe als +hamil ton +ma yo +we ave +arrang ed +whar f +occu py +b vb +as aki +ot ter +nor m +vi es +de tox +tion al +dere k +id ad +ad missions +constitu ency +u pper +woo t +allo y +se ve +lu b +un comfortable +ed win +ab re +d wight +ar che +virtu ally +sp ol +pri e +ai i +er r +swit ch +bar ack +se ok +cou l +wn t +pou l +o live +caffe ine +cardi ff +notor ious +de mp +ex cess +bar r +t ford +a jay +bump ed +my thology +shel ley +fal con +shakespe are +must angs +no ted +bon e +civil ization +sy d +par sons +un official +hy ped +sp ends +oppo sed +v ings +space x +noti fication +deci ding +bio tech +out si +sal ah +! . +fe d +ss y +c ms +bad gers +cr o +ela ine +n ba +dy our +n ant +honey moon +climb ed +conom y +ath a +m ell +ne bula +nature photography +juli e +bm x +inve sted +mon o +lieu tenant +wat kins +techn ician +o se +ka e +ì Ľ +mc queen +pre ach +trav eller +flexi bility +ze bra +reta iler +p ant +ben der +brand t +squ id +war rant +veri fied +cas s +pier cing +hon ours +t ying +mor ris +kis sed +op rah +panor amic +me i +splat oon +wich ita +ari as +gal li +indy ref +good times +athe ist +confe ssion +ow ski +re pping +ad ditions +mechan ism +z im +j ans +su f +cho pped +beg innings +vitam ins +ãħ¤ ãħ¤ +or th +po les +ru b +antarc tica +indie film +web cam +ket ch +bre tt +cle ment +her on +defe ating +hydr o +buc ket +wand ering +sid ney +future of +b inge +on ies +knock out +administr ator +syn the +l ent +jan i +bar ley +premier league +ner ds +cr m +bra s +bot any +evol ved +rot ter +ro wed +tum or +weal thy +Â Ń +mon arch +li shed +da hl +ðŁİ ĥ +bu ch +ken yan +Ø § +red ness +assemb led +se mit +hud der +shro p +ran i +lear ning +mor y +iti a +geo graphic +worl dof +f b +pho sp +boo gie +am ped +? ... +che w +dwar f +ar us +s sen +ru sty +recru its +h k +gar de +app lause +vol umes +invol ves +ta c +hand bag +trans late +ffe l +se ym +aqu atic +trans fer +zo di +and r +acade mia +cr ater +te z +ar se +adap t +col oni +snow man +mal i +hang in +di schar +oy sters +pho e +colon el +w ba +hispan ic +thri ving +sh y +ag les +sales force +cre me +so les +la fayette +â ī +ter ia +ach a +sp erson +go go +car ly +the ore +am ore +vo x +af t +ãĤ ¹ +stap le +mu ffin +di agram +ino x +su stained +av ent +me ta +arbit r +dec ay +ado le +Ð ½ +ec ol +ph o +n k +o cu +gr anny +ç a +luxemb our +stad t +alber to +le vit +am as +d x +or phan +co bb +as c +lo gy +immen se +chan ts +off line +p ent +bre x +w inger +plan e +i el +nichol s +ca thy +nar uto +low ed +/ // +ignor ance +cat astro +you ts +sch en +buil d +haz i +s ine +critical role +du g +dete ct +lo gs +en amel +stpatrick sday +ed die +co pa +cigare ttes +ho ff +kay a +la goon +ra pha +air borne +choo se +puer tor +ke v +gui ding +fro sty +bor ough +mir a +ðŁİ Ĭ +cade t +anu sh +yo gi +e ger +fl ing +slo pe +nin th +we ston +foot wear +f n +may weather +a am +pla in +stair case +witne sses +work outs +ro bust +dex ter +co hort +ðŁļ Ĺ +sp ell +ha ze +o om +organ ising +wild fire +cont acts +av on +min o +upd ating +ðŁį » +li thium +ing ual +k is +au ga +lo com +de duc +u da +th ak +boy le +mp er +hot tie +eri k +re vised +is la +travel photography +oo za +en qui +confe rences +clo ver +g room +cur ves +live on +per f +displac ed +bo log +xx xx +ðŁĺ© ðŁĺ© +te al +ve ssels +rain forest +cal ci +pan ther +gira ffe +ta sted +imag ery +pad res +day time +bas s +ri pe +opio id +nu e +vin yl +invent or +sen s +process or +mu t +gad gets +bibl ical +shann on +jacqu eline +car y +the resistance +ali en +n vi +co sy +bi har +fo ley +ren d +mu gs +fa ken +cl one +ni allo +gra bbed +chi hu +power house +n tt +chero kee +spon ge +imple menting +rh ine +le one +ðŁį Ģ +pret tiest +infra red +impro v +swit ched +tu bes +con tr +bl k +projec ted +be aver +yo t +bbcra dio +thi gh +per secu +apologi ze +w ack +po ster +oli ver +az a +lou d +( ?) +f the +women shi +spar row +blu sh +us able +sc ales +it ative +peu ge +ne eding +legg ings +glam orous +mat ur +c z +wat t +da b +tam ar +et sym +bau er +heart felt +h n +else where +bir ch +alu mini +hu ck +e me +j l +traf ford +d z +por tions +ana sta +arthr itis +esp n +ber gen +viol ation +yo shi +c z +northumber land +clo sures +ðŁĩ¯ ðŁĩ +smi ley +r w +tel ugu +inten si +gre gg +ve ga +dun geon +south bound +ba il +domin ican +semi final +chap ters +h itch +van ity +trans iti +recomm ends +sati sf +bar ca +queen s +( ( +de struc +stra it +ra vi +dess erts +in tru +har am +k os +fo e +fat ty +pais ley +magn itude +dri dge +com ey +schem es +vision ary +our t +down loaded +ðŁĻĮ ðŁı½ +gd pr +lan i +p wc +gu ad +nic est +stake holders +re ferred +george town +arvind kejriwal +schnei der +in doors +all star +strand ed +gen der +ze pp +ma sses +ðŁIJ ± +pati ently +bl dg +z ab +we arab +vi vid +he ck +d ella +sy mb +je opar +la ger +à ª +comb ines +ne c +br ay +flo p +tx wx +jo ys +pon t +pro found +sur round +mad hu +ma ble +ay r +te as +n sa +open ly +er nest +ãĥ © +to po +g na +anti oxid +ti an +e tr +c ello +ma thi +gener osity +b iting +man ic +kel sey +chee ks +ten der +w th +pron oun +ultimat ely +gu sta +ari anag +ger ry +ble ed +red dy +mic h +mitsubi shi +oper ated +sex ually +ma u +cl lr +vi ds +co c +mel ted +ðŁĮ Ī +q ld +ite ch +instru mental +end game +ðŁĵ ĸ +ener gi +brow nie +tam il +at in +domin ated +pra ises +fire place +sens ational +men a +k arti +un prece +ru pt +ori ental +mc cor +tour naments +scen ter +re eves +prescri ption +sam e +fra u +tru ffle +em bo +roman s +bla sts +techno logical +pr at +b sb +y ar +tren dy +ac l +al ad +ðŁį ģ +o hh +bankrup t +tho ven +regar ds +is er +war wick +vine yards +real m +niallo fficial +do ta +ge mini +to do +v able +¨ ¨ +la u +wre ath +ju ve +nat asha +le ver +lor i +hor ser +cc tv +air bnb +es anders +sin clair +ema biggest +high school +con test +optimi stic +t te +ðŁĴķ ðŁĴķ +ss d +ye e +hel ena +con sen +ric ks +jes se +an ic +ðŁİ ¯ +re acts +ro be +independ ence +vol tage +m ington +s ant +à¸Ļ ภ+-------- -------- +sentin el +ke tt +rehear sing +aaaa aaaa +sof the +stir ling +sear ch +wi gan +stand out +sna il +pent agon +Ä ģ +ch lor +cru st +net any +chemi st +disapp eared +ric ardo +sp iders +bo se +war ren +me ssing +bann ers +gu el +par ach +ma id +coun ted +epi le +bon fire +speech less +se tter +meas ured +rejec ts +nik ki +le ster +foren sic +fab rics +alo ha +pre served +wat ford +deta iling +dar th +bo u +car ly +... ' +tail gate +noti fications +å ¤ +pas sive +trous ers +balo ch +ro ther +typic ally +à ¥ +sp it +wi z +sic ily +technic ally +ex pose +st age +hu bb +cre am +cap s +po ke +sle ek +ju ne +tempor arily +de z +awak ens +l ame +_ - +ji ha +tues days +advis ed +advis ors +exi sted +dis agree +news room +lo sers +world tour +dr ying +al di +har ness +foot print +hobb it +p mln +i ro +que red +asse ss +gaz e +sa b +th ian +í Ĭ +ti f +ob serve +ev il +dra wer +swee p +cor y +co dy +kyo to +cal lum +n inj +lau rent +be i +sket ching +custom ized +du r +regre ts +knox ville +ìķ Ħ +mess aging +grac ie +abun dance +bi dding +bre wed +fl ouri +therapeu tic +alt itude +ho gs +bur ner +elec tro +wonder fully +he ater +post pon +li very +r all +ad as +a ac +sau l +brook lyn +play house +âĻ¥âĻ¥ âĻ¥ +char itable +in y +z ah +compet itions +be av +plu gged +o is +do om +astron om +speci alized +max i +ta ps +cellu lar +depre ssed +folklore thursday +cri b +e mul +ë° © +fi gh +ru z +car lisle +spe ar +side walk +de i +depend ent +lac es +nh s +ðŁĮ Ļ +reali zing +net work +ric he +re gin +re fresh +st ral +pa thology +pla id +psyched elic +hin d +u ka +algori thm +lin king +progre ssi +fe y +d ade +hydr ated +b ant +fam ed +cot sw +bo ise +as c +rac ing +ja vier +ww en +mar lins +poo p +swe pt +toni ghts +we f +ani me +slo vak +âŀĸ âŀĸ +cla us +lem me +cli ppers +re ls +arianag rande +r te +ko t +thal apathy +hungar ian +zu ma +y von +is u +jour neys +clin ics +be be +ww f +n ws +super heroes +er it +sle ague +identi fication +mo tto +ba i +sour ced +ill er +ap i +pri se +unprece dented +dam as +tuni sia +dra in +undere stim +e ther +quarter ly +rewar ding +al ham +wolver ine +cab ine +hyp no +nad ine +hav ana +da e +ðŁĵ Ī +dr on +read ings +b ati +pic o +mer ci +iti an +wal kers +el ope +mi key +god zilla +bur lington +abu ja +social ism +at ility +sh ell +harry potter +g no +ab ur +re leg +fel ici +ro gen +neuro science +inst in +ath am +vou chers +j arre +fu se +def ici +monte rey +de port +mid day +pp ard +fre ed +ame ter +wil t +n ingham +pr att +liber ty +slo gan +o to +pr i +co ated +c pd +ne tt +il las +mal awi +evol ve +accessi bility +ðŁĶ¥ðŁĶ¥ ðŁĶ¥ðŁĶ¥ +or nament +b p +el is +son line +chi ro +fl ick +ib m +ar ak +en ables +gar land +san e +cu ties +tri p +rotter dam +n ys +lam ps +lu cas +bo g +ra ils +travel led +hic ks +en u +sab ha +scru b +hi er +hart ford +fo o +fer nandez +tre vor +mat tress +appo intments +ale j +fe i +o logist +saf ar +oc ta +sr c +sha un +ambi ent +dri c +bi ker +she e +must ache +h ta +bo one +her ty +car dio +bra kes +rec ital +consi sts +overwhel med +cau l +robb ins +im it +al th +ur l +bi bli +on ne +black livesmatter +diffic ulties +tel ang +tall er +ðŁĵ Ĩ +deb ating +bur rito +mo vember +strength ening +bo e +te stam +mirac les +base ball +re nee +ðŁijī ðŁı» +al fa +âĺ ĺ +unstopp able +ec s +g mo +giftide as +path way +fen cing +ðŁİ ¤ +b ham +ra s +sk o +d led +thel ast +magn um +bin ary +wil de +wil der +wh ati +barbe cue +h ism +can oe +kur di +eli ve +advant ages +mad ame +bi er +mis sing +enter tain +air force +y ama +c is +hash tags +j is +ve il +dream y +ten se +may ward +ch ateau +hunt ington +âļ ĵ +v all +up on +bl ouse +dun es +ðŁĺ ´ +fert ility +m ole +curren cies +st u +ber lin +toa sted +div as +wal t +lar k +por a +hit ter +um er +chil led +bal ancing +fa is +y in +or tiz +east enders +h ate +ur al +ap ril +tim el +à ± +per o +sto cked +respec ts +th t +best friends +giving tuesday +be ad +inv ent +im i +nap les +comb ining +tok ens +thir st +ma sc +par rot +sp u +dent on +* -* +t res +subur ban +wid th +si ve +con tender +siri us +lo k +troop ers +outra ge +tur bo +frag ile +me ssed +do h +disc ord +netany ahu +re sign +forgi veness +mo han +mun ch +cam ou +identi fying +enab ling +hot ter +thorn ton +jai pur +ar ya +ðŁı» âĢįâĻĢï¸ı +mu staf +maj ors +o ke +du ffy +roh ing +til t +ðŁĩ®ðŁĩ ³ +rock star +she ep +hend rix +ra v +in vention +do u +lagun a +gru mpy +sw is +im pe +) ' +you ths +bun ker +st ache +oppo se +indi es +acceler ate +ml p +ed en +w ann +k ail +akshay kumar +su pt +pol ym +midd leton +extra ordin +wil son +australi an +alumini um +way ne +alum nus +mat ics +gri m +er nie +opp a +competit ors +rand all +h ence +decla res +pre aching +sha he +can e +sustain able +stap les +le dge +ad ena +doctor al +bur gundy +decor ate +ren dered +ri sen +pr ank +di or +bee thoven +flo or +ac com +to t +ho dg +touri sm +say in +objec tive +mar kers +premi ership +en abled +camou fla +gi ant +Ñ ģ +smo key +ric ket +pan g +de pending +s ation +evol ving +inter cep +cen sus +tof the +re en +mendo za +trum pet +marke ters +an it +ðŁĻ Ĭ +north western +v la +foto gra +blackand white +che wan +wi g +tro om +ginger bread +k n +ro mero +n fc +or chi +fun ko +sour ce +f s +ra ped +o st +tar ot +ann ually +ðŁĺ ¬ +r ill +del av +.. !! +se s +can n +medic are +ph el +ape x +guardi an +rema ined +r pm +a ñ +story month +instag ood +neighb our +p ing +sem ite +my stic +as cot +mat er +hand ful +dang ers +ti d +ana heim +opol y +sh allow +nami bia +tor ia +procu rement +big bang +announ cements +prosecu tor +beng als +sal le +en roll +ga stro +sugge stion +ba k +ha ul +budd hism +berni esanders +flu te +fati gue +cyn thia +cho i +ir win +gu a +str ous +h p +ba p +satisf ying +play a +ðŁİ ¼ +inst ap +al ice +t p +irri gation +ðŁĩ¬ðŁĩ § +in tric +clu es +ple x +sa x +he pat +dump ed +signific ance +by u +medic ation +pro v +tough est +corn ish +âŀ ľ +kel ley +u v +si zz +si bling +me st +di stor +diplom atic +aun tie +b hat +son ic +bren da +pump kins +ro ch +black burn +ur ged +shi a +arrange ments +floo d +sa unders +lec turer +nou ri +popul ations +diplom acy +consist ently +ðŁ¤ Ļ +t mund +cauli flower +l ily +vocab ulary +vari eties +coo ker +up town +qu ent +mo sa +re inde +velo city +spru ce +social medi +i ber +volun tary +proce ssed +bal tic +y ang +leban ese +d p +dol ly +arrange ment +y uri +cran berry +kal yan +elev ation +cli ff +pu shes +ìĬ ¤ +sil ic +co wx +eter nity +sla ves +vine gar +glou cester +con tained +breaking news +aga inst +renov ated +norm andy +hero in +ys m +mo ds +gre ek +un di +tren ch +v h +encoura ges +head ache +gr ange +: ' +ever green +Ù Ĭ +reck on +ab used +th ru +cho ice +ti dy +col der +scho ice +ha in +bru m +li ars +bre it +yor ker +sh ack +he idi +micha els +sco pic +fasci st +play ful +ca c +yas ss +sh ad +.. ? +qu en +ram irez +clif ton +pr s +best fan +âģ ł +gener ating +head set +disappo intment +abstr act +bo iled +paren thood +azerbai jan +exhib iting +bom bay +oli vier +ko so +un lea +mat ernity +iz er +si ves +r hu +col l +saskat chewan +fre akin +de k +na g +stab ili +ðŁį ķ +organi zer +bo sses +ar u +u va +at able +ta un +after wards +fert ili +ver ge +az i +mor ph +๠ģภ+jer k +cosme tic +ko w +stru st +ap ache +post cards +for mul +ì ĭ +spin al +jack pot +elec tri +Ã Ń +lo y +gra der +diab lo +ar di +he sit +f w +arch ery +pa sh +the ories +repe al +re live +per cy +âĺ Ĩ +im in +syn chron +sham poo +coup ons +o to +la i +thou ght +luxembour g +mo v +ðŁĺ ¥ +ge mma +se ated +m ga +strat ford +un certainty +shi fts +est o +fo ol +fire arms +cor rie +ki ki +appa rent +p ills +olym pia +fi d +elev ated +de cks +ignor ing +av alan +ro v +whist le +p tsd +milit ants +robo tic +pac ers +quil t +bankrupt cy +lic h +per cussion +celebr ity +al s +( ; +su t +pokemon go +h g +off s +gibr altar +scre ams +billi e +gen ome +mar in +be ams +arch bishop +em in +bedro oms +g ated +ol ly +warran ty +at own +cudd les +gun na +k ic +vi ve +cy mru +nar row +pro b +le o +refe rences +manufac tured +cho pper +brun swick +sem is +don ia +r ye +man o +hur ting +? # +hol li +investig ations +c els +ðŁĵ ŀ +le ster +temp les +sto rey +mc mahon +toi lets +wo of +ï¸ İ +le verage +at om +night mares +victor ious +haun ting +custom er +ag i +yo ongi +mon ty +ver onica +w ur +inti mid +blan kets +volu tion +j m +âĺ İ +am on +jud ith +ðŁĺİ ðŁĺİ +distr acted +dri p +hurric ane +and es +revel ation +tro op +ab leg +col lin +tibet an +wor rying +inter nationally +eat er +camero on +brad or +y uk +ðŁĴĹ ðŁĴĹ +tra k +slo pes +ci er +ne a +ol er +ta ka +albi on +volcan ic +am n +a fi +ob stac +face time +ger ing +n pr +metall ica +organ ic +ðŁĴ ¡ +ki dd +d ances +pemb ro +wash er +m its +om er +emo tionally +tan go +ip o +do cks +scan ning +spec s +tho m +the ology +emer gen +om i +g pa +selec tions +un necessary +ima ge +ter s +induc ed +gi gan +rent als +supp lied +m fa +shan kar +lat er +pa jam +cla ve +Ù ģ +ma hin +carl son +avi an +ano va +kati e +aj ith +design ated +chocol ates +investig ators +gla zed +prin cess +er ry +ra gn +ou rable +hr u +sun dance +peuge ot +steam punk +gh lin +gre ase +hi res +z ap +per ce +j ill +tom e +he hehe +joy ful +mae stro +ni shed +gene alo +v ich +p its +fox es +good man +emer son +lo bes +con verse +o ats +thom son +ra him +mal ware +ah i +man kind +re sin +im g +sw ood +kin der +sc roll +ar a +sak ura +ro bbed +xi on +ny a +c ism +ce dar +be in +mour ning +tor to +heath row +done gal +bar b +hydr ation +k or +elim ination +su pdates +hill s +appe ti +star red +ko m +gw en +dd d +cra y +sc anner +personal ised +seren ity +re design +meta ph +box ed +judg ment +no se +ë ¹ +er ad +ac ne +supp liers +ener getic +v om +as ap +ðŁĶ ¸ +ir vine +hat ch +la ss +ad ren +waff les +accur ately +ici o +itt le +se un +occup y +web cam +thene w +ent es +ga i +j w +accoun table +vis or +ir rit +licen sing +hudder sfield +gen ie +ðŁİ ¾ +atmo spheric +ten sions +spart an +clif ford +ol an +north bound +ame en +cen sor +u el +ster y +$ $ +far rell +hy ster +cl t +se dan +rep lied +descri bing +micro wave +sla b +pro sp +assi sting +ru bio +e than +hh hhh +gu ay +z man +ra ise +roll ing +o e +n ile +ambro se +scar borough +hero ic +coo ks +mor t +chop ra +ðŁĮ · +to b +shav ing +stac ey +dor m +motor sports +wi ki +fol ds +sp iced +stress ful +liter al +fu dge +pe ggy +wa ite +tre sses +se sh +pr ic +ðŁİ ħ +fri ght +r va +mumb ai +po m +tt v +cel lar +tom e +andro id +dor is +tsun ami +tin der +o ec +m wc +dor tmund +no thin +l iti +so u +believe in +at u +kno cks +mag ni +ss sss +ro hit +ine ws +ang i +m andy +ke ttle +intermedi ate +av ant +cur l +endor sed +ori o +ur t +consider ation +wi res +shel ters +b ino +vik ram +imple mented +ly dia +bu k +paro dy +c news +under graduate +canu cks +sam i +polit ically +ro tten +gh z +tex tiles +over load +moder ni +recre ational +fli r +bat on +typo graphy +ov ation +intrigu ing +pilgri mage +al ge +ad ays +tcm party +sp elled +cur ls +boo ze +ste m +ann es +ir ls +spon ge +sho pper +sig nation +bra ss +mi stress +le ah +beg inner +lau derdale +augu st +pre school +ta ping +tai pei +execu tives +b d +rhe tor +esc or +immun o +deeplear ning +stat ues +it us +manu script +ly ric +cor vette +mol ly +la ge +de p +cn bc +le st +je ssi +fi fe +griff ith +oppo sing +ran g +dr ills +respec tful +p ity +d ell +har ding +play boy +blo ke +shut out +k ili +o sp +se attle +bc poli +mis es +journ als +team ing +es ther +fre ddy +Ķ ï¸ı +metr ics +no tre +gar ry +for ty +navi gate +perio ds +bened ic +j id +da w +ance stors +restor ing +con g +aller gy +tit anium +c ence +lean ing +ab bas +v ast +uc f +roof ing +e man +seve rely +vo gue +ve au +in bound +d z +tane ously +stret ching +man chester +dr yer +dav is +kan th +the game +it ted +re tain +el les +conge stion +frat ernity +ol lie +lo ki +fre ely +cho o +pon y +sc ep +tab ly +bal t +rock n +di me +lo gging +ðŁį · +ad u +ha voc +water ford +char is +swee tie +run ning +ner d +erdo gan +z ara +weigh ing +fif ty +pre cise +low ell +kurdi stan +r yo +or th +syn th +lin ers +phenomen on +art illery +il legally +constru ct +nostal gic +gar th +al ta +shel ton +a sean +w ander +dur ban +di versi +bon o +cl on +le man +sh un +obstac les +appet ite +fe eder +respir atory +di xie +formu la +an to +so ber +extin ct +au c +ing les +legitim ate +; ; +min nie +ipsw ich +dram atically +ðŁijı ðŁı¼ +ingh am +milit ary +mon et +us navy +for k +dun no +play er +q otd +st oo +ex or +ethiop ian +film fest +pe red +c ate +sau di +in ner +sin cere +tion ality +ale e +de eds +cooper ative +ir onic +cro cod +br ary +post season +cam per +can ary +e in +exten sions +nb d +sher wood +spo kane +hu mp +jit su +ê ¹ +dar yl +p si +stab bed +offer ings +expe cts +cav al +body building +fr aming +f ca +ye arly +bom bed +sk il +resear ching +jud iciary +gree ted +tu dor +mil o +innov ate +ðŁĺ Ľ +r hs +ru by +contribu tor +fam er +soci ally +m lin +fi ery +ut ter +beau t +it os +de voted +rain bow +bar ney +pe ren +ar jun +r na +gab by +ut i +hann ity +pick le +ser v +qu akes +pp e +fe m +wh itec +j n +victor ies +ðŁ§ ¡ +gol fer +congratul ates +resul ting +mechan ic +ur ve +cen tered +kie v +an s +in cub +< < +c mo +bestfan army +dap h +en ham +on cology +ku sh +t xt +ori ented +fashion able +c sr +sa hara +r ack +pd p +han son +ภĩ +ti ers +ra r +pan am +in sky +sa hi +testam ent +asth ma +in her +fisher ies +or der +ho we +gall on +ep is +suz anne +drow ning +paneli sts +ðŁĺ ² +ë ¦ +al ach +commemor ative +at tribu +ðŁij » +mo o +visi onal +week sary +gu st +ak in +poin te +ee e +di spar +ni pp +dent al +st all +pi an +bor e +ul ster +tic k +ir r +tae hyung +micro phone +bermu da +ga ard +el er +plumb ing +hu gely +âļ« ï¸ı +race way +cam bridge +mar cel +burn ley +to ast +holly wood +fa sting +me red +hib ition +ca pped +benef icial +ow ning +cont amin +arab ian +to on +cap ac +hul u +sm ir +nutri ents +se in +graph s +con ditional +ðŁij ħ +or ac +play in +nor the +tor nad +mar ian +ju mbo +lex i +incredible india +road to +uk one +confu sing +sp h +shan k +pi ed +mq m +positi vely +sher ry +path ways +consi ders +tof u +argu ments +resil ient +che tt +with dra +ter o +ated ly +sw ana +he b +fli ght +har ley +decre ase +kind le +book shop +³ ï¸ı +marty rs +sm ur +mc cl +concer to +sti me +rejo ice +app lau +cle ment +mer kel +jai me +im mortal +isle of +mar co +youtu ber +stal king +me too +st ack +sp ouse +u st +lu v +âļ¾ ï¸ı +eque strian +ev ing +fl in +nick name +the big +as ar +st acks +wal ker +bor a +kidnapp ed +hur ling +humb old +rec alls +co pper +ann is +se o +mer ger +mu ir +ad dy +ðŁĴª ðŁĴª +be x +cr acy +con an +congratul ation +mid st +âĻ ¬ +for bi +op tic +cr ate +crocod ile +mad agas +secur ing +ast on +o gue +savi or +salis bury +love it +fuji film +cast les +as st +ar rows +sp acious +tr s +poly vore +progre ssion +m ri +nel son +bi m +indic ator +o da +pe pe +re signation +gu t +sne aker +log ically +az y +are lla +te aring +jo shi +ssion ism +q pr +mari ah +p x +ble ed +mi an +med ley +we iss +ker ry +gat ory +at al +madi son +av enger +nab y +pl and +gi les +fresh water +d ington +ta j +demonstr ates +n tv +bul bs +sunday morning +pe ake +souven ir +wa h +ton nes +m kt +complex ity +con den +ross i +b ing +y ds +su k +n go +mid land +ol y +life is +ri pple +mo reno +dd ers +tu s +á ĥ +bou l +x a +hol dings +wn y +shadowhun ters +ke i +asp ire +m ous +ow en +so ak +skir ts +moun taine +stor ming +ch rome +ri ots +sar ato +amaz e +less ness +nav ar +crit eria +ra fa +indul ge +ay er +por to +nam o +........ ........ +yi elds +val le +j h +mac ron +sa ins +dur ant +tra ilers +wo t +confeder ate +sh rin +id ol +form ally +ten e +motor cycles +than g +no de +bang er +dal y +p ats +enroll ment +au ctions +at al +ar bor +lo gos +de arest +trans action +dom ingo +fle a +ser mon +de ck +sin cere +questi oning +juli o +was p +pre tz +armen ian +k ham +inflam mation +picture sque +acci dental +film makers +ðŁĺ ļ +ðŁĴ į +ca sey +so b +yee zy +good will +parag ra +ss ly +fe ather +dy ed +assassin ation +na de +b cs +app lies +femin ine +fe u +ext ent +depu ties +l ack +psy chic +go i +kill ings +pse u +ðŁ¤ ª +un c +mar l +tan e +mck enna +sur fer +influ ences +free way +hack ney +mal aria +el and +te au +rema stered +Ø ± +raz or +gg y +cor ro +lak sh +fla ir +honest y +hoor ay +de pp +am c +wedne sdays +q a +ed its +- $ +se villa +dou bled +human ities +c cot +som os +r ine +af a +si oux +re construction +wel ding +th reads +am ish +encoura gement +po der +bo ck +bal m +p tions +stand up +accompli shments +guar ding +convic tion +ac ion +napo leon +depic ting +att ack +su i +wear able +âĸª ï¸ı +pot ter +esc ort +vis e +to ts +bo on +event profs +angu lar +womenshi storymonth +bar row +sch i +ac comp +ti k +l end +kensing ton +wol fe +st acked +cra shing +exhi bit +wing ed +sab rina +ma sa +k ms +alway s +et t +pla sma +counsel ing +pick les +nfl draft +mr s +inev itable +coura geous +staf ford +writers life +ho s +e j +gh yun +trade mark +adri an +influen cer +coron ation +ra ging +explo red +usa f +excep tion +eu x +tan ker +sw ami +pac ket +ðŁij¨ âĢį +f en +she en +a ero +j l +re gal +nw t +au ster +meh ta +char ge +a ste +b ate +inf eld +racec ourse +collap sed +fle ece +z il +al lie +alternati ves +geor ges +ðŁĵ į +quir ky +fc b +nat geo +philanthro py +bra i +every day +ðŁIJ ° +ach ers +ja an +fin es +q i +fisher man +distin ct +gri mes +nation alist +comm ence +ro wn +âĢ ³ +z ing +f ter +hr w +baro que +bl ender +kitt y +hoo ks +c ited +w anda +consen sus +reinde er +an and +supp ly +me ds +v n +ol ph +rat chet +shel don +secur ities +ë°© íĥ +cro m +mosqu ito +j eric +im mac +dimen sions +â ¤ +di ssi +sponge bob +dami en +steven son +jo anne +del ish +yi kes +than x +surve ys +postpon ed +alco holic +al ised +ðŁĻı ðŁı» +do ch +sen tim +mered ith +com pares +b ago +happy days +mo ss +ãħ ĭ +ne c +gn ment +frustr ated +comb in +ri v +ec lec +col lo +compli ment +actor slife +ct to +nic ar +op hon +apar the +man t +ja de +trol ley +optimi zation +eye on +eco logical +qui st +ep he +ॠĩ +cin co +appo ints +old school +c pr +behavi oral +min aj +:- ( +tag ging +ev al +jo aqu +ðŁĺ « +ha k +de me +jama ican +so s +hy att +hand book +libr arian +hanni bal +pump ing +ch om +f man +ga i +hu ll +respon ders +green ville +n us +vau gh +ðŁİī ðŁİī +ta xi +gold berg +man tra +te ase +forbi dden +metho dist +ati vity +* *** +ec t +mc gr +Ħ ëĭ +se b +amid st +disapp ear +thy ro +phili ps +er ina +v icious +stream er +million aire +ma p +str ick +hack athon +gh a +ed ic +mi ka +pe ck +ill i +anto ine +ar ca +op tic +ma ure +ðŁĩ¦ ðŁĩº +cla shes +man ly +âĺ ģ +al var +and res +me i +el m +ww ww +al tered +l te +ê¹ Ģ +mo jo +for rest +thal ai +non t +spee ches +acknow ledge +ign ite +x factor +ðŁ¥ Ĥ +mead ow +disru pt +debu ted +scrim mage +pharmaceu tical +fi dd +found ations +philosop her +et al +publi shers +bo ys +c ke +ru gged +opti mism +re be +phil harmon +nar cis +ral lies +lu is +go blue +fol ded +un acceptable +optim al +li sa +pol aro ++ . +en za +âĿ £ï¸ı +mon opoly +grace ful +dair y +du a +diffic ulty +judge ment +o si +mer sey +flu x +new found +ter ns +dimen sional +in vic +al ba +am it +abudha bi +alger ia +autom obile +the ad +lo tion +acceler ator +vac ant +iti on +lu f +al ic +pl l +bla zing +ba z +sen e +ðŁij ¼ +villa ins +direc tory +eis en +to ck +broch ure +ri pp +hb d +zayn malik +nic he +lo lol +certific ates +mor se +fac up +x ham +un wanted +im ports +carne gie +fan sign +mo u +r alph +destroy er +sw ing +trek king +cili ation +pit bull +g aps +ho well +defin itive +mc le +f ps +et z +bol ly +lyn n +gan o +at ure +fur suit +co il +na v +but ts +tro jans +eu re +en ko +sch umer +horri fic +install ment +br b +subur bs +a bel +vi r +de sh +cun ningham +ðŁIJ » +span n +sch we +ke mp +tr u +ste alth +qu es +le w +deli ghts +ko ch +hu mili +cr iti +il t +sp ells +mi ley +car ic +ðŁį ´ +lc fc +substitu te +oun g +? !! +af fir +predic table +class of +er r +cy press +chand ra +age ing +__ __ +ther land +don caster +el in +yo shi +sail ors +har ris +jo anna +niger ians +h ers +pla gue +pro cra +k no +can ton +busine s +un h +pra kash +c in +bow en +co ating +m als +be gging +smith son +ponti ac +sp ies +dam ian +pl ine +und ant +al ta +one ss +shame less +da q +bb m +wal es +stam pede +ser um +Ù Ĩ +cataly st +x n +ab sc +free zer +ch un +ari os +mc cre +fore head +he ars +damas cus +tac oma +ardu ino +encoun ters +stan ton +lg b +ab as +" .. +ke te +drac ula +ele m +g ne +zepp elin +la brador +pul p +op tional +or n +russi ans +san itation +hil ary +etsym ntt +pen alties +au st +ig ans +olympi an +medic aid +vers ace +va pe +re stra +pe ep +sexi est +st alls +di le +the a +punjab i +pupp y +tuesday motivation +ðŁĵ ļ +the flash +roc ket +mo dest +chihu ahu +on na +k sa +hur dles +ca ve +fail ures +sp lit +bo ho +gur l +disappo int +ho ward +nug get +fran z +stal ert +kaz akh +for getting +sch ri +ag ate +am at +eve rett +du et +veter inary +juli an +ch ills +bra ve +ghost busters +lan do +gre ets +profit able +d é +ti r +ze e +om en +pd x +gray son +har i +fix es +stab bing +swim mer +symb ols +compli ments +po se +func tioning +th nx +gi r +corpor ations +bar low +lo e +off season +distin ctive +marvel ous +nik on +enri que +ky u +ja ws +amo to +lom bar +travel blogger +fa h +ouri sm +tri stan +so e +ce ase +ðŁı ħ +z ac +mck enzie +taxpay ers +swim suit +bl o +les ley +kan sas +w ks +ki el +provo king +my les +str ing +kangar oo +galac tic +fif th +s ke +we ir +ll is +mat ory +ðŁĩ ¿ +un ci +re productive +roo ting +ti des +gad get +.... ...... +alex ander +bow ler +scre w +apo log +eri ka +wal ters +shet ty +lan e +ban ter +as ant +me so +v ain +" "" +us i +fer din +accomp lish +man sfield +bom bar +collabor ating +cla p +it ure +s da +smo ky +na k +im person +car la +com ra +bur gl +lo co +ti es +in hi +trac ey +se is +diss er +rr rr +dra y +prote ct +cor ona +hun ger +ck en +c eli +trou bled +predat ors +fic tional +shav ed +riche st +metab oli +ful ham +gro oming +mono chrome +wa sting +as co +ast e +ti sta +remedi es +ung soo +south end +perman ently +bu mble +procra stin +ident ical +practic ally +ma scul +su ke +assu red +val erie +devi ant +grizz lies +thi er +pur a +ne pal +not ts +bil ateral +spo il +car mel +cine matic +ph l +ni fty +ma o +hypo cri +la ser +pan try +mathemat ical +el isa +coordin ation +bel mont +a it +radi ant +bo iler +man g +f ag +cr c +h ams +br in +â¬ĩ ï¸ı +famil ia +âĿ £ +sab er +ru pert +gg an +rit z +mic h +sal ford +le vi +gra l +ðŁĴ ¤ +n ino +ce d +business man +ul tr +sim ply +compre ssion +pa ins +hal t +ë°©íĥ Ħ +landsc aping +n f +croo ked +er d +itt in +ddle ston +sur passed +ino a +da g +bl en +exten ding +at ing +al gae +ball er +u mar +snoo ker +col lu +flo wn +thu b +ridic ulously +ki sh +op le +di re +as ser +ari sto +sc iss +h ating +trou ble +syl via +suc cul +plo ts +sincere ly +al er +laure ate +br ack +att n +rif les +me to +collec tible +cu omo +conte stant +consist ency +ant z +rang es +abig ail +de b +mini ster +grow ers +an oo +hoo ver +dream er +nu cle +resear ch +mi y +sha hid +ma v +d honi +cin i +do j +hin dus +part ying +dal i +alon so +inform al +clark son +it ton +ki an +cit yo +mor i +la sted +as pen +libr ary +susp ici +qu at +den ial +fol der +ch ori +swee ping +eni x +ðŁį Ĥ +Ø Ń +nas car +handmade hour +mou l +heat wave +em er +exam ine +ib n +gr ind +po v +tion ist +m bo +she ila +integr ate +om es +take away +cer v +con nie +tic ket +ce led +bi en +visu ally +madagas car +sor ry +gu i +park run +tra its +la be +pois oning +ॠĢ +vi able +bohemi an +denti stry +bad os +spr outs +mask ed +te ddy +ðŁĺ · +sa f +sa as +ji ang +ti ght +spe aker +withdra wal +bc n +as signed +class rooms +fle ming +ðŁĴ « +super girl +tot als +table top +e books +horizon tal +cra z +flu sh +j ard +c dc +er son +ãħ ł +green wood +ni h +co x +ad a +lit re +go ing +v icky +cur ved +lou ie +gra ins +hy e +lon ge +reme dy +tra inee +san jay +super stars +ma ser +man u +s age +wh l +ðŁĺĤ ðŁĺŃ +ðŁijį ðŁı» +m sd +en z +rab hu +j oo +gh u +ac er +e po +resurrec tion +justice for +bl ended +mo da +avalan che +france sco +re spective +g s +ye ast +wel ch +devo tion +ge tin +athe ism +am ic +carol yn +lo c +ld nont +ave c +us da +le gged +bra very +b lower +cow boy +he h +sti ble +buff al +chann el +run chat +âĺķ ï¸ı +ide ology +best seller +y oo +pe anu +bon ne +fel ic +edi son +fr actu +naren dra +pp ets +seym our +ri viera +he ctor +necess arily +bi anca +soci eties +the best +w g +sent ences +win k +vacc ines +pal ooza +jam ming +as f +mp us +agre ements +ec k +ba c +hon ore +com pul +wild cat +im posed +yo ga +hud son +can celed +l ich +fu zzy +es que +ch uk +w vu +se k +fli pping +r hon +wi shed +wh a +cap ability +len ovo +ìĨĮëħ Ħëĭ +vi vo +tv d +nor a +sil k +pas adena +yo semite +valu ation +clo cks +u ber +mr c +dar kest +au bre +ss o +bell y +wrest lers +kill in +lou der +buck ley +ge el +ad on +un s +appe aling +ðŁij ¯ +semit ism +list ens +fit z +ãĥ³ ãĥ +ny lon +ar ty +seem ingly +hal a +su ited +et y +she ds +mu ffins +ap ric +um ents +u ta +jam mu +chelse afc +star z +yo ko +roo t +clean sing +di ar +pione ering +ihear tradio +dig iti +fin dyour +can o +ðŁĴ İ +z ol +spac ecraft +six ers +moi sturi +b ile +ti sts +hor ton +rang ing +colum bi +mete oro +senti ment +ep l +foo th +text book +drain age +r ly +sc ue +imran khan +ðŁĴ ¸ +margar ita +ed dy +predic ts +gamer gate +advis e +growth hacking +love you +ug and +v f +beng hazi +s later +ne wor +ch el +independence day +p np +cul len +hoo dies +num bered +brit t +t sa +kl tu +s ages +mom o +onep lus +col l +gu ts +w ta +mesm eri +enh ancing +chiro prac +j is +teen agers +m one +constell ation +sweep stakes +e ze +slovak ia +la ye +pear ce +wa ver +po gba +k ron +sur geons +mar x +ti d +gg a +desc end +p ours +upri sing +wal la +sab bath +bachel ore +mack in +k am +peter borough +hor a +ðŁĮŁ ðŁĮŁ +think big +r j +hy drau +sp al +univers it +ðŁı ī +mail online +league of +ten ants +w ally +lan ce +heav ens +dd r +bol ts +am ir +i phone +ci gar +en du +re i +el abor +r inging +john son +characteri stics +sal oon +algori thms +tal kin +m tn +di ve +region als +ff ice +hat i +deviant art +so tto +shir o +l ama +k we +f aded +por ting +tu mmy +est ates +buen os +ðŁ¦ ģ +beli ever +pen etr +dar n +sp ite +can opy +fashi oni +t illa +pet als +eli jah +bra wl +marty r +ë°©íĥĦ ìĨĮëħĦëĭ +mid town +eric h +d apper +sm town +me gam +ww w +le le +on s +cat fish +fir th +fossil friday +ball park +th aw +pot ent +illi e +cre ep +car p +so ap +gun dam +infe c +yy yyy +ठ¨ +z ag +rit t +calcu lator +bo ca +ok o +to ad +threat en +refin ed +olym pic +accompli shment +bacter ial +a ji +tat um +feli z +she ed +j at +th ic +jam al +ðĿ ĺ +lin a +ðŁIJ ¯ +jo king +yot po +pin ch +ak ron +her b +motiv ation +li a +ho stage +cre ek +gam ble +russ ell +patt i +fo tos +c pc +bro ken +back the +cla ys +u mm +stock ton +mat ernal +ü r +la kel +cent ury +be k +infe cted +ภ¡ +smack down +man ned +ta hoe +sm es +bas a +su la +augu sta +. * +rohing ya +gre ed +counsel or +silhou ette +gra vit +cla use +' - +bo bc +occa sions +now adays +dic tat +be ard +n ally +brigh test +kab ul +inc india +dhan ush +archae ological +che ape +mizz ou +d hi +ov ski +bax ter +asse mble +à ¢ +gi gi +ac am +wis ely +haz ard +north ampton +âľĪ ï¸ı +me th +bla sting +re unite +mu lus +ali zes +t read +mil a +ed ward +ko va +pe sto +ðŁij ¶ +vit z +hydrau lic +refurbi shed +mo tel +isab ella +hom me +sever ance +uph ol +mis erable +f ari +lat ter +ef er +crack ers +es l +ac io +yy j +in an +ec b +z ind +pan as +tru cking +re ed +sh aker +burge ss +em pire +ag nes +n ington +art works +fr s +ti le +bi ome +eu n +ch ong +americ ana +god father +go blin +i shi +! ). +temp ted +gen omics +mand ate +ck y +ðŁĴĻ ðŁĴĽ +som ali +br andy +in ven +spoke sperson +pc b +yu an +h g +fa z +starwar s +ro wan +blue grass +don g +d day +trin idad +er ton +ban ning +re tention +cu red +tober fest +re set +we is +deta ched +behindthe scenes +immun ity +ph a +bra y +ðŁij ½ +ran cho +ram say +est onia +nd tv +] . +cab aret +tar o +d v +show cases +plu m +ðŁij ¸ +son oma +pre pa +memor ab +e stu +drive way +u les +magn us +x r +nn n +much as +en ge +stre amed +fore stry +audio book +tro y +reck less +kil om +ru ler +ra k +proce ssion +i ons +po ole +noc tur +wh s +farm house +per a +par me +hypocri sy +s ics +v ant +cas k +holi stic +au st +Ð ¿ +in do +ðŁij© âĢį +di so +disp atch +ol sen +make it +en nis +cent re +ar range +ðŁĮ ¼ +sal ted +ea siest +f ate +reg atta +mo zz +ac an +sin i +g ically +ch ops +chick en +work in +ha gg +invol ve +wee ds +book day +wake up +ky r +michel in +fu ss +re juven +vac ancies +incar cer +m st +sc ents +sovere ign +kick er +à § +bo d +âĢĶ > +sa h +mob il +shrop shire +oph one +dress er +mis suni +hep burn +i mo +foli age +diagno stic +as san +cycl ing +guil t +c sa +puertor ico +win elover +wake field +do ggy +k he +pa pp +co g +al lot +cu ck +poe tic +mi o +re vit +mag ician +ç ¥ +ant enna +west wood +mber g +lux e +oat meal +Ø ¬ +te at +ffe e +sear ches +l ly +plu to +el on +let tering +inno cence +fa i +ann on +telang ana +ma it +neu ral +can ni +ar oma +a stor +fe x +co cac +mon etary +f ent +un sure +' @ +indi rec +teh ran +isol ation +li bs +make up +merce des +ff y +he tero +de o +sco m +cur sed +veteran sday +franken stein +shre ws +de co +ge ese +lefto ver +ha did +vari able +acade mics +carol in +under going +vari ation +na h +ssi er +gamer sunite +pur suing +emer ged +ll ers +control ling +ro aring +mete or +vol t +daw gs +be aver +is life +bathro oms +aci onal +pre vent +lake district +in als +y ani +gra bbing +sac ks +le z +sw ay +k ool +time s +klo pp +la de +con cord +resul ted +revi ve +recon ciliation +ol and +az z +gir o +mand arin +de en +nutriti onal +is coming +van i +aw www +der ived +love your +stop the +shou ting +nov ak +ðŁĻĮ ðŁı¾ +lo af +displa ying +sunday with +ma guire +ch eri +ðŁı Ł +re match +qu ic +Ú © +y in +ðŁĺ ¹ +ili ve +z ip +our ke +down loads +sw at +missi ss +care rs +t ment +proper ty +hahahaha haha +gi bbs +sur rey +ar ise +tic ism +sti a +ir ling +fro g +co se +bas sist +fore ig +lea u +pil lows +hol la +eli e +disclo sure +peanu ts +inte ch +ww c +plun ge +trium ph +cor i +sli ppers +ðŁĻı ðŁĻı +neutr ality +ma re +hair y +gang ster +hu mming +cust ard +mer lin +ale a +s by +dam p +mo han +ver bal +j st +gu tted +b jor +un finished +ðŁĩ¯ðŁĩ µ +un happy +âļ« ï¸ı +by pass +at su +fis cher +sa v +afric ans +re use +mid way +demo lished +ger rard +her cules +Ä Ł +medic ines +cl icking +sur round +jo ong +wav ing +tri bes +wet lands +offici el +argu ing +l le +do va +su zy +club house +ne gro +ob tain +ga o +gl ance +assi st +ch os +ãĤ ¢ +âĺ ķ +adri d +occur s +st ans +par don +livel i +emplo yed +re visit +ff xiv +bb le +ne aring +min er +ðŁĺ ¹ +giov anni +up to +mar vell +mar se +to wels +cb n +engine ered +y elling +spart an +si ans +ðŁĻĮ ðŁı¼ +se v +coyo te +sta di +t cm +app en +shenan igans +open access +so aked +ma squ +le vine +stro kes +l k +aparthe id +hipho p +char don +may may +ha asan +stri pped +fr o +scri ption +f ton +h f +pri sons +marsh al +ķ ãĤ +an cho +com promise +classi fication +buzz feed +bblo ggers +deser ving +) / +s way +ob o +camp ers +poder nfamily +p oured +bri e +squir rels +se ize +: # +le k +ti mb +st acy +nas daq +repe atedly +br at +mi ghty +competit or +mah one +de si +o ke +bm w +shi e +f cb +cheape st +minim alist +par amount +n ate +har as +insan ity +lat eral +ment ality +mo zam +ta pped +yad av +u sp +b way +the od +bil t +ra ids +em press +adap ted +pat ron +nut shell +ag ra +be aded +sundaywith marsha +vi king +proce ed +main tained +thinkbig sundaywithmarsha +sn es +mus ica +to wer +ch ab +bo k +sm t +insul t +harve sting +windo w +ru ther +be ige +dec al +indic ate +ma iling +ri ft +po le +ander son +ch oral +sp ride +l ili +ev elyn +imrankhan pti +.... " +ke red +un dp +water falls +se ars +le mans +world series +ri el +ani e +app ar +score rs +lam p +a than +phys icians +qu inoa +refu sing +vu itton +unle ash +s la +pat i +shou ts +inten tions +fo amed +europe an +neighbor hoods +me er +man son +du h +br at +con es +bow l +kazakh stan +ठ¿ +in appropriate +del hi +ketch up +ful ton +s ys +consul t +gar field +to go +f ml +f led +b ds +facilit ate +ree bok +selfi e +elev ate +activ ate +bi ble +ca wx +b ys +cam ille +sy ou +sk ool +her t +w bc +ple dges +recor der +po sh +ac re +so aking +mat il +v sco +shoot ings +pla r +e con +ðŁĻĮ ðŁı» +rashi d +u bi +ðŁ¤ ¤ +sw inging +wi pe +rap tor +m su +music video +dur ham +at tic +apar ty +fe tus +activ ation +aa z +motiv ate +ðŁĴķ ðŁĴķðŁĴķ +j al +ठ® +ag on +sche er +stal ker +fo ster +az zo +tele gram +vi gor +s laugh +screen shots +entrepre neu +kri stin +inten tion +ch illi +fr action +don a +ge a +tc u +s ite +la k +em il +d nt +bor o +wil kinson +re cu +ato day +t anya +bl anco +cd n +brilli antly +g cc +ac c +evacu ated +ther ine +den ny +cait lin +she pard +pou ch +hand held +sou theastern +ha a +à ´ +re solutions +led ger +sr in +r ar +shat tered +chim ney +im with +mete or +hand led +ra ke +town send +en han +shi py +duc t +tw x +inflam matory +war hammer +theat rical +gro s +sk ar +sco tty +ni el +tit o +tin i +conne ction +_ . +goldeng lobes +sha q +ðŁı ³ï¸ı +hall way +fron ts +effec tiveness +gla ston +d hs +ex pi +to h +c pl +sc s +re o +ha g +resemb lance +hor an +abu sive +qu er +virtu e +cho lester +a q +shan e +m ce +carri ers +di stress +re wind + ¡ +voo doo +int act +ann o +ðŁĺ ¤ +pi led +adi a +ãĥ ³ +en ow +di gs +light ly +goo fy +turb ine +governor s +con te +re open +pa h +i ve +cra fting +swee ps +jo di +an de +zu cker +kaw aii +o ko +v ai +out line +kri sti +ts n +insp o +qu int +fil thy +lyn ne +listen ers +depar ting +or d +t weed +, & +ale k +sel fish +nor ther +recogni zes +i ps +be s +a ed +w ills +pe at +surround ings +mon uments +ais le +be cker +la v +quant ity +v ah +helicop ters +tu cked +alv arez +sha pe +o bey +ad diti +road side +m ite +bl ers +ep age +j au +ignor ant +b ins +lu lu +x o +c fo +ee eee +apprentice ship +shef fiel +to i +ho k +faken ews +deplo y +aid an +husk ers +ãĢ İ +west brook +mi ster +confi gur +car r +fic a +proceed ings +ha w +ste ak +mur derer +pay day +a jo +p vc +don ates +bi af +nom nom +be it +k ali +x rp +ahmed abad +se mic +che y +x tra +an twer +head lining +squ ares +roun ded +flu ore +bol d +disa sters +am oo +gener ic +cran es +brief ly +gi g +auster ity +anticip ation +for ti +treas urer +cann y +ce cil +dete cted +check list +ภ§ +pam ela +bar bados +an field +hear ty +tx lege +peren ni +arro g +ing ram +âĹ ı +ty ne +spo on +r ation +am ba +m be +cam el +h hs +york shire +reflec tive +fre aks +to k +ju do +partic les +du bs +ban jo +accred itation +prover bs +over dose +inte gral +gu ang +mc s +super car +af b +al vin +ail s +x tre +st aging +tw ent +rabb its +mar o +inste m +dol l +cr ay +sant ana +ble ach +mini ons +che ap +man t +di vers +catal onia +lo is +mat ri +cou gar +kay ak +e gre +p so +a ia +å ® +char lton +tr acked +sc ari +pe tt +f wd +x in +gra vel +br ic +bigg boss +ar den +hu gging +pal ms +st v +li mb +the movie +handic ap +ri me +z ai +stu b +indi a +lithu ania +rhy th +p ita +maced onia +high ered +brid get +schwar z +ske let +hi kes +ant arctic +c ps +mash up +Ð ° +n ell +chand ra +he ir +an us +sher idan +mi mi +muse u +bec ca +an ir +bar rie +dioce se +compar able +ðŁı³ï¸ı âĢį +yuk on +me p +hor mon +mer ic +al f +con quered +christ church +ðŁĴĻ ðŁĴĻ +hazard ous +poo h +cont ing +retro spective +par ame +na ir +con sor +ho tra +astoni shing +cater pillar +u man +ti sm +t vs +serv ic +croy don +mor ales +c g +cu m +te ur +scan ada +s all +magno lia +el ise +th our +à® ¿ +ag omez +phel ps +ë°©íĥĦìĨĮëħĦëĭ ¨ +wh os +weav ing +si sd +pro poses +cro ws +pre sale +econom ies +bernar do +sha hid +air show +mc cann +hor ticul +nr l +du el +mongo lia +tou lou +requi rement +struc tured +ed i +o lives +he a +cu ter +Ð º +enthusi ast +harri et +domin ion +sub mer +ðŁį ĥ +sa ab +nes burg +mo ff +def ended +bur t +rewar ded +gold man +op tics +khali d +house holds +buc kets +ce cil +che ss +substan tial +ef l +oper ation +evalu ate +st n +rece ssion +l ll +tom as +tru ths +ak bar +s words +p act +embarra ss +ha o +ay urve +scrip ture +ny cc +op t +di ameter +sc ented +organi zers +re lat +ha e +dream ers +de se +ðŁĮ » +restric ted +n ale +r hp +dol an +mun ster +ha ired +consult ants +jo ints +hu mil +d ill +relent less +t é +af il +ut ilities +japan ese +condem n +pet ite +colli de +q f +peach es +cou rier +l ore +âĺİ ï¸ı +reli ability +ch uk +ðŁĻ ĥ +stu res +ge ther +ho stel +bi er +- _- +â ĩ +e ze +ta ilo +di ent +blu ff +chu ffed +pil ip +mon arch +e em +bu chan +b ick +op au +ku ps +ภ¢ +pist ons +sp ins +m and +ce st +bur ne +v ile +cher ries +bec kett +need les +pan ch +ë Ĥ +haha h +trou bles +insi sts +do you +g mc +mor tar +deleg ate +in n +g anda +sin atra +ठ¤ +spee ding +pu pil +pre mises +ali gnment +pi kach +as us +j alan +Ø µ +lime stone +fol kl +parme san +ce il +mo y +shawn mendes +ac up +hu st +ot es +med ina +ma di +gta v +censor ship +ar g +swe eney +sy kes +col o +foot steps +cann ed +adv ance +gta online +healthy living +ðŁį ¾ +a ig +p ality +oc s +he brew +im minent +berk shire +jeremi ah +out going +bak er +entr ata +ma ids +gro ves +bo c +a del +m fw +con science +arm ys +nut ella +conte stalert +novel ist +la h +ban ker +marque z +ðŁı ¡ +to ff +out age +gr p +ðŁĺŃðŁĺŃ ðŁĺŃðŁĺŃ +musc le +du dley +nvi dia +mi di +m uni +ess ays +dat ac +car ter +ภ£ +t ans +i ves +public ations +al er +ok wx +il u +cu tt +har p +out law +luther an +br ill +bo lic +do well +green land +be sties +path i +pay ton +gue st +har den +ðŁ¤ © +ann ed +evacu ation +po ised +mc der +b han +o i +envel ope +ci d +ca vi +ta pas +book review +grey hound +âĻ ª +fe ud +lun gs +for te +rai der +ff er +oni x +dep end +yn wa +rel ating +de vs +ðŁĴ IJ +acqui res +d ha +j yo +priv ati +can ine +k b +cra b +sar din +imag ining +k j +em por +down hill +ne z +ta eyeon +nick imin +gb p +à µ +w ap +sec co +ma shed +ðŁĴ¥ ðŁĴ¥ +augu stine +diss ol +dic tator +â ĵ +vi per +ed fringe +vau x +hard work +book let +no x +chi ff +ðŁĴ ¨ +observ ations +xbox one +u sher +ke er +lu p +dal las +cal gary +ma dra +di ous +k bs +wood ward +hero ine +lu mber +sea world +o ws +mc ke +maver ick +gu la +cross roads +fan g +s ade +nik ol +chee tah +me c +pp g +er ick +ðŁİ µ +tox ic +bj j +viol a +sp ire +ch ino +tra vis +institu tional +ha as +low ry +w ac +ea e +hu mid +mp ton +ru ck +je w +c ine +zim mer +se f +bhar at +fre es +aam ir +ðŁĴ ħ +z inc +wan e +multi player +royal wedding +e el +preci pit +qu ery +kimber ly +isa bel +ful fill +ig an +vau l +pan e +sc y +dig it +gun n +u tah +dog day +fi on +xia omi +da c +el ast +cha vez +ro blo +g ine +ten th +ab h +ke to +hur dle +na dia +memorab ilia +ha bs +qu an +h w +hv ac +pix ar +ec cle +kram er +accu ses +ðŁĴļ ðŁĴļ +per se +mean time +wa hl +atle tico +âĢ¢âĢ¢ âĢ¢âĢ¢ +ott oman +no vo +k us +conne cted +tru sts +d mv +spen cer +rahu lg +do ve +sto kes +bolog na +enthusi asts +à ª +rockstar games +ted cruz +du ras +s acked +late x +immer sive +cer t +lu cin +princi pals +fa res +sa ils +far n +am ent +saf fron +quent in +check point +fer ris +ex cur +ðŁijī ðŁı¼ +bai ley +se h +ter re +mad am +s band +wan derers +cumber batch +yy c +digit ally +blackandwhite photography +roll in +moroc can +ðŁĮ ħ +din ner +d well +to om +m ye +ez ra +cp fc +war hol +me er +jon ah +no aa +s gate +so on +secu lar +g ating +ti o +dri ver +si ssy +assan ge +ta th +ed mund +bobc ats +ra ji +po stage +stu ds +m gm +kat o +edin burgh +meet the +shir t +fa a +mens fashion +sp reads +wi m +car ts +phoe be +j ars +bot swana +Ù Ĥ +ed war +sk ar +ri ve +gu sty +c tv +ferdin and +su therland +nickimin aj +k v +si us +bee ch +re z +desi res +on ial +camp o +quar ry +lor raine +gil more +ig gy +µ ï¸ı +ho pping +avi z +ðŁĮ º +uni sex +dedic ate +att itudes +ste er +jun kie +rail way +y b +whi sper +key an +k us +ju g +di x +a ins +sum mon +ov ich +sy ed +her ald +ma ison +me ded +wild flower +main land +ri sky +ru kh +over looked +ki c +destro ys +nam an +ki p +z ano +champion sleague +ban dit +quin cy +smi le +cal vin +open ings +ta pp +ol ulu +spec tro +accred ited +ap k +pra ised +bar nett +pol len +premi ered +selen agomez +tou red +screen ings +uu u +mis o +en se +adam lambert +guel ph +har yana +hu tto +le ar +l tc +po ached +brex it +æ Ŀ +tt c +pa vement +mon gers +ro e +ad ers +ling ton +particip ant +ca red +ga il +y ates +lan tic +dash board +jo o +feli pe +ssi onist +bu m +s end +a eri +thu gs +luci fer +a he +dete ctor +fil ly +gas oline +ham per +hump day +the ta +the band +fore casts +o hhh +lo bb +hol l +cp u +az u +ad ar +hai ley +bu b +car t +quo ted +an archy +pan cre +twit art +al den +st ash +the less +or ni +belie bers +mor mon +partic le +avi ation +⬠Ĩ +webcam toy +sad dened +cru is +ham let +n ct +roll ins +marque e +saw yer +reli ance +a ura +di ec +soo thing +sig nings +ak is +à ³ +at kins +aer op +ðŁĮ ¿ +y ab +sh ari +con nol +du bbed +manufac ture +convin cing +feelthe bern +ra u +pu lit +on ec +gem stone +ur ging +bag u +ga h +aci ds +fi anc +zodi ac +sn oop +her rera +initi ated +ven ge +profess ors +pro di +stron ger +e mission +bb a +hal le +ta pp +haw an +wh im +compe ted +myr tle +ir port +cold play +ach e +ske p +m son +ss ic +calli graphy +swim mers +me y +pp c +thri ft +po c +re places +commu ter +âģ¦ âģ¦@ +go ers +lo gue +para dig +bas kets +sensiti vity +joh an +atl antis +& & +suit case +anxi ous +l h +str i +gal loway +stre ad +war den +gr ounded +ffici ency +li feat +reli c +disgu ise +island ers +f cofficial +classical music +b mc +en field +bi que +oak ley +bat man +sla ying +ner ves +mul tit +calci um +projec tor +scott sdale +ant ino +gri ps +kim mel +des mond +prote stors +hi atus +metaboli sm +conclu ded +press er +ti pping +sli de +e to +hun ting +aus open +ri k +pp ery +innov ators +pitch ers +ag ger +fun gi +z ad +proli fic +rockn roll +bl ames +ct ar +stam ford +q ad +mozz arella +insan ely +den ver +ph ouse +nom ad +ï ¿ +s ris +pro du +hen ley +pag an +am trak +ru bi +in cl +tu tor +sco tia +wo es +sing apo +fun nel +turn bull +know ledge +gri mm +real madrid +we are +missi les +con sol +emo jis +sne ak +smi ths +ru iz +br ou +i el +ha ver +ðŁĮ ļ +kin gof +basil ica +circul ation +prin ters +ta pping +ri dley +dra gged +ha j +writ er +fundament als +personal ities +me tre +stereo types +bur le +best of +n ffc +ha th +mini stries +a ali +trac ing +pav ed +ł ï¸ı +g ic +insp ire +tu g +ha re +repe ated +ex pon +lol li +rho de +pre cin +install ations +instag ram +az ar +i es +sole ly +du kes +mission ary +van guard +fursuit friday +on d +pol ari +ma st +har an +jos é +jack ed +ec oun +al ities +ne ph +ra vel +moder ated +sco w +s fb +uru guay +as o +ni g +au du +p ints +lat ina +ben z +m itting +char ted +mat ology +cit ro +biop ic +ðŁij Ń +djo kovic +fox y +agu il +so to +an ada +sin king +sc rap +hair s +bethan y +fact friday +ðŁIJ IJ +unlea shed +) ( +contra dic +ram on +coast line +y ong +sn sd +li gan +p ome +mit age +ge tt +wat i +ri sk +so aring +bru sh +f pl +av an +å Ĩ +lar son +sh ear +mul til +blu r +multi media +chun ky +par i +n ani +weir d +cholester ol +char les +dream ed +tan ning +puzz les +fr am +hand ball +ch ag +beli ze +al u +bang s +Ñ Ħ +detec tives +mc g +ish q +bo thered +saf c +mp ing +ten eri +g ays +sail or +an gi +mul ticul +gue ssed +ros é +high ways +bro om +chatt anoo +- ' +see ker +on ed +at f +lu c +> < +bar i +per cep +jewel ry +as ph +sor row +sl ing +mam moth +jac kie +ë § +wilt shire +sa o +can cell +im paired +tor ial +bre ed +guy en +jud ice +tit le +pro spective +applic ants +ðŁį Ĭ +epis cop +e id +b yo +stock ings +ðŁĴĥ ðŁĴĥ +ll p +sna g +keep it +l ough +ol son +matur ity +!! !" +cop ter +i sha +bl i +wil mington +tr youts +th ai +ðŁ¥ ³ +pe bble +kra ft +f p + º +ssi vely +li vin +contest ants +tex tures +jo an +h dr +film festival +prov ence +wi do +op end +c si +sto wn +cro ati +ad just +host ile +analy sts +il an +cu ppa +bru m +newfound land +good win +me tt +mall orca +plu gs +bu k +bb hutto +wrest le +sa ire +sho pped +for za +le head +vi vo +ba st +ro xy +reg is +hard working +hon olulu +desp air +young sters +ni g +impro mp +roll tide +de emed +tre ason +ru shed +for ged +ff f +pikach u +bri ggs +do it +ac cent +la us +gla ze +compet ent +a ho +photo g +mid field +le go +har vard +min orities +re illy +slic ed +once upon +initi ally +financi ally +landscape photography +har dro +qu o +mm ers +par kinson +smu gg +read iness +bru tally +glou cester +mp ed +bbhutto zardari +mur der +ye d +dat aviz +sr t +dow ning +bi ans +m ü +fle ck +fli pped +s ly +brilli ance +ri m +k um +bubb a +ko i +knit ted +sor g +ma is +ðŁĮ ² +ti ss +su stain +sen su +ak han +zi est +exam ines +chardon nay +user name +short list +re bs +on o +dar ing +hard wood +che que +righte ous +light ening +dir k +shra dd +du ra +down stairs +sh al +ami gos +ru ff +s law +ri es +red nation +man us +ðŁĩ§ ðŁĩ· +distin ction +u bun +dur an +mi gra +thi ans +la ver +domest ic +k x +jaz zy +justi fy +belong ing +insul ation +color stv +drun ken +chann eling +qu and +xi ii +enligh ten +kan o +fati ma +teen choice +terri fied +p ba +as ley +met museum +dun e +pack er +ki o +ðŁĴľ ðŁĴľ +bo iler +fas cism +ar mored +back grounds +in mates +embarra ssed +defin es +th d +we go +silic one +lo on +el ding +bor rowed +he mp +ak sh +kaw asaki +br y +de af +kill er +dispo sal +ðŁĩ ° +glaston bury +un covered +o xide +po ff +d ant +k j +ku ro +dri zzle +peop les +fe e +pro pri +dd lovato +pi ggy +ot is +aller gies +u bis +pengu in +ser a +vi z +prosp erous +ici des +tornad oes +sene gal +web cast +sto red +enchan ted +bb cone +bay area +entrepreneu rial +rednation rising +experim enting +ang an +lot to +they re +por e +er p +seren e +east wood +bro kers +bar ge +stal lion +timber lake +tailo red +dy stop +b ate +lat ors +di xit +bran son +dynam o +ky lie +shame ful +bt wn +spring time +mix ture +s ounded +lu ton +dad es +mal a +op ra +en ic +rahulg andhi +se wer +~~ ~~ +ky u +nor theastern +ca er +bc u +nir vana +kitch ens +ous y +al m +river dale +hid den +fl int +sp d +pat rons +katy perry +au gh +exhib itions +sm c +shu ts +at ore +da in +some thing +ber th +bo g +por ter +gen to +con cussion +ang lic +ro we +gr illing +scar lett +master ing +mor nin +comm ented +si me +si zing +christ y +ce os +st m +at ry +tari ffs +vac ation +pre judice +p su +paren tal +far age +can a +cap com +koso vo +you re +men stru +stal in +grape fruit +br an +che sa +dav en +exc el +!! ) +๠Į +distribu tor +ce a +bride sma +millenni al +wa in +ob serving +mis ery +plan etary +expo sing +bra ised +comp ton +don gha +q l +spring steen +th ul +syl ve +cab o +pal ad +niel sen +gaz ing +ba ja +r oud +orchi ds +johan nesburg +se man +d ji +oper ative +affe ction +eclec tic +at c +mut ant +aw x +nic e +mel bourne +indu lg +tu lip +dias pora +wel p +big gie +mississ auga +retri ever +or an +tam my +c ta +hipp o +seas oned +ger mans +eng v +marvell ous +im f +rela ys +mon tan +maur iti +me ister +as surance +reig ning +su fficient +han e +no thing +pos se +nav y +in love +brigh ton +en qu +ch ung +sweat y +es c +cal ed +man s +nicar agua +sl ices +mo cha +washington post +bb n +dam ned +grow ing +en burg +lo an +me s +wh oops +believ ers +spi el +vo daf +l at +s led +cricke ter +brown e +golf ers +bar ra +wat chers +lu igi +sw amy +mom s +pit ched +san tor +cr s +si re +sc amp +bo de +ste war +jon ny +ent ity +pac qui +mind ful +min india +bear ded +temp t +scorpi on +eat on +authori zed +ar to +s vp +op athy +cch ini +house music +disney world +âĢĶ @ +pro pose +di y +expen se +ten g +pupp ets +sm el +d aca +per ry +fin n +boo sting +lefto vers +cou gs +satell ites +man y +az e +g ong +fi e +metho do +fer ries +ðŁ¤Ķ ðŁ¤Ķ +explore rs +load er +attrac ted +il ton +godd amn +pi azza +doc tr +sav ing +paragra ph +visu alization +may ors +work flow +ack les +ðŁĺĤðŁĺĤðŁĺĤðŁĺĤ ðŁĺĤðŁĺĤðŁĺĤðŁĺĤ +ठ¸ +twer k +clu t +lo ver +te ases +si an +o te +deter ior +accor d +l fw +swar ovski +nat al +tra ps +k ina +analy ze +laye red +bever ages +un it +ran som +pe shaw +dest ined +astro logy +si pping +miley cyrus +cam ino +marshmal low +bli ss +out back +fa q +int oler +humil ity +po ppin +hallo ween +mon tene +op hy +nu n +tattoo ed +a as +ðŁĮ ³ +dale y +qual ity +du sa +fisher men +swi f +ter rac +st au +le in +trol ling +ship ment +garden er +march madness +head band +gr t +bur nett +w and +!!!! !!!!! +gh e +du x +hu d +war ner +ðŁĩ ¦ +ex ile +rescu e +rat a +d han +duc ati +dro wn +bl ends +spi e +alli gator +simul taneously +broo ke +u ke +k har +comm union +ri ka +ford fc +chin atown +you rown +me y +can al +syste matic +de pri +ox ford +an il +w ut +equ ation +be z +fle ur +the good +lang ley +ad ity +ed ith +al fie +о ÑĤ +en cry +br ill +ex emp +ce sar +mb ling +ab ri +sc icom +j ing +school ing +mi ka +mechan isms +impromp tu +rhe a +moo re +crime a +be sto +wri ght +el ders +ro ds +kam al +folkl ore +be et +mini on +reli eve +thr o +team usa +pas cal +made with +boli via +itt i +free bies +desi red +best selling +l iness +la den +ke ane +mi sts +hipp ie +atta chment +@ / +se w +flan agan +âĿĹ ï¸ı +supre mac +stl cards +si as +q u +rh ys +ste ep +val leys +v w +pav ing +disp at +al ison +por te +id u +new sc +soc ket +mo s +co star +re vo +prote ins +stanley cup +m cal +ear ring +se cs +mc lean +cap ric +nick elo +ad en +v c +shou se +adap tive +maxi mize +entertain er +pro se +gri ffi +six teen +lam ar +mi rage +saudi arabia +awe ather +ru st +in filtr +fashion week +ðŁĺĬðŁĺĬ ðŁĺĬ +selec tive +bubb le +a den +fen nel +deci sive +m ta +mock ing +mb les +st amp +mu le +bernar do +gr in +po tt +j ingle +vet tel +colom bian +cam o +motivation monday +ba han +p ly +dh ary +k ami +x men +sleep er +gar a +my sti +confi dential +conflic ts +p neu +ce s +insur tech +clean se +me rely +va is +tu x +the great +shar on +ma j +hol a +eco systems +aj ay +aa j +hu sh +har mon +backto school +wiki leaks +reflec ted +ðŁĺ ĵ +commemor ating +ac et +buck ingham +messi ah +tu ous +hor net +to be +d q +he ine +mi g +pl ate +nichol son +sp ie +cumber land +nor mal +pho bia +happy halloween +city fc +mc el +gilli an +ke to +lu de +de mise +su ga +str ate +mcgr ath +visit scotland +foo led +cb r +gc se +col ori +po td +missuni verse +fin ances +ma poli +for ks +Ø ´ +cann on +medic inal +ðŁĹ ĵ +kh o +wre ck +pan to +bag el +gu ll +syndic ate +ic y +pr c +ki en +zi ka +ti sh +pe ta +c co +li za +ch ut +ex traction +el g +gl i +fu eled +pos it +respec tively +leice ster +br ink +vulner ability +im ported +e sha +ðŁ¦ ħ +r ural +re ll +gam ing +atlan tic +aband on +no ah +re solved +pro state +aller gic +ps d +âĺ ¹ +dun geon +fang irl +illumin ated +m hs +white sox +d ently +ck o +endor se +over ly +dazz ling +prior iti +night life +ut il +be have +flam en +east bound +ðŁĴ Ł +ilove you +gov uk +mozam bique +alle gi +dr i +testim onial +ath s +ì§ Ģ +mm y +shab by +pro secco +friend ships +cal am +dam ages +off set +jura ssic +jun o +arre ll +ðŁĴ © +interven tions +dare devil +car ver +run away +ran e +truste es +ha ute +dep ths +ðŁİ Ń +me in +sacrific es +con cier +ne sting +i zzy +me tam +ilove my +ur ine +du lu +mal hotra +ve ins +night ly +co at +an di +he witt +lon el +ci ble +wr ite +jen nie +sant ac +ĸ ï¸ı +str ato +singapo re +sop rano +kri sten +cheer ful +flee twood +fa iri +m eli +wa st +tur nt +sfor sale +sc rolling +angel ina +ren dition +jeric ho +nick y +or b +fla vo +patri ot +ash eville +sick ness +re fund +aggre ssion +b pl +ãĥ ĥ +elu sive +thi story +hang er +bu ffs +vil las +at kinson +sp h +ja it +decl ined +wo k +supre macy +oo tball +ey ang +ðŁİ ĵ +s ford +ath i +consu me +road ster +e so +u pro +reci pe +au f +uc i +ar on +oo oh +cs go +re ich +mc d +min ute +ladi es +pun k +rut gers +mee k +ariz on +ta j +land lord +de gra +autu mn +lyn x +us f +b hi +fairy tale +dongha e +bet sy +explo ded +chen nai +op a +pro tag +br ant +ðŁĵ °: +g f +pal li +ðŁı¼ âĢįâĻĢï¸ı +su t +ill ini +colum nist +shir tless +de centr +sear ched +ec or +bu ggy +s ack +ðŁĺĤ ðŁĺŃ +de t +ther i +or naments +bring back +to v +quarter finals +ic he +con stra +gi er +buchan an +vi x +kay aking +mu stread +swal low +mel b +sc af +op al +may oral +har at +ðŁ¦ ĭ +schedu les +id f +ha gue +ro z +a ah +d mc +du plic +ca che +orph an +frac ture +rec on +ch av +bun nies +al ain +mustaf a +ðŁİ Ļ +vac ations +dynam ite +tex ted +broad caster +ðŁĴ £ +ste amed +rock er +di etary +luxury travel +inaugur ated +sa wards +vaugh n +lincoln shire +click ed +kra ja +f anc +remo ves +layo ffs +mc far +bre eds +win nie +jon ghyun +incen tive +vari ations +pat ton +atur day +persist ent +pr un +pi ers +dal es +æ ĸ +breast feeding +r ance +ta wa +Ĥ âĸ +mur doch +cap tive +thi stle +nic a +commod ity +cou ldnt +board walk +graci ous +practiti oners +n gc +scru m +ner o +camoufla ge +col on +he i +phys icist +saturday morning +ten er +si won +colum ns +bru ne +y vr +ba ir +reti res +hal am +cab er +shaz am +min u +cas cade +milk shake +gri d +d ren +vin cent +so dium +plat ter +cheer leader +chen ko +y ak +elimin ated +ty po +y man +re think +âĿ Ĺ +ts ville +bernardo kath +ex tr +ðŁĺģ ðŁĺģðŁĺģ +ta o +re per +mo ths +em powered +c iting +transpor ted +mon ks +san at +cle ars +bachelore tte +camp bell +racha el +har le +hand ler +climb s +inter ference +rele ase +sh and +r bs +hr h +ãģ ª +val le +r é +sli me +w akes +chu bby +slo an +el ves +ath en +attor neys +micro scope +ston er +sc aling +o be +c out +se man +mid week +bal sam +ðŁĺį âĿ¤ +ti ful +v ish +lo tta +ri pping +re mn +ti re +le ap +ha vent +la by +hi mach +whisp ers +we in +ðŁİ ¸ +wild flowers +se le +u cc +li ability +az ine +sw ings +k ya +ta ir +re main +e do +flo ps +poc ket +grand ad +exam iner +gr is +ffe ct +ðŁijĬ ðŁı» +stud ded +heart beat +de acon +firm ly +infec tious +ste f +out lines +le asing +cla ws +sen se +tab s +hoo t +mo sul +spa wn +co a +hog warts +ve in +alban ia +manu el +b ino +vaux hall +scot land +go bucks +mat ty +phy sio +tor ino +const able +investig ated +s lower +mistak en +bay er +wild fires +vo ic +x on +time to +chas sis +bar ric +pi on +bald head +woo k +regi str +dra fts +b hs +li gue +l ick +staf fordshire +baf ta +dar ry +je anne +ven ding +cor p +⼠³ï¸ı +kid dos +fen way +ca o +west bound +ðŁĺ Ļ +dv r +quick er +bla h +goo die +ðŁĴĭ ðŁĴĭ +vo x +esp er +fac ade +cor relation +red bull +rou p +decl ining +chi ve +mc gee +tur o +in der +f eller +fu g +il ysm +mar di +peshaw ar +ki eran +ine ma +meat balls +pe ck +depre ssing +sen sing +gi z +dd ington +spring watch +ro aming +yellow stone +horse shoe +am man +week day +ol or +ðŁ¥ ° +boo sts +spr int +scar ves +je e +bee tro +cl an +all the +ìĦ ¸ë +enlighten ment +ado be +re generation +? @ +cont ag +yach ts +to u +mor a +en voy +r ani +go li +dhanush kraja +wood working +streng ths +se di +disc s +ar ina +sc on +lit e +ano ther +ðŁ¥ Ĭ +ye men +gu ern +sav vy +lo yed +biom ed +heart break +comra des +milli e +pat ch +un f +jar vis +bl aming +commemor ation +ge y +å ¥ +cardio vascular +alig ned +docu ment +. ? +aesthe tics +em u +the irs +le h +ps ic +si f +pl ateau +ex pend +domin ating +rob es +mauriti us +excep tionally +hom er +discover ies +bra un +ten nant +insul in +ðŁİ ® +car bs +te as +? !" +zi e +franco is +brow sing +th ol +cla rence +hel per +ob tained +cas sie +le es +! , +pome gran +hu bs +presti ge +] [ +mach er +bott led +pun ch +pi pe +o ch +gall ons +deliver ies +u ra +un day +mon de +depic ts +re gency +outra geous +khal ed +car o +he arti +za g +develop mental +over coming +stati stical +flavo red +for ds +cre atives +lau rence +di as +sun screen +in ked +pre acher +n ul +impac ting +auti stic +âļ Ķï¸ı +o ss +pel icans +cele ste +v b +ru mp +mc gra +fair fax +hu mor +bbc news +row ling +cal der +seam less +ag ne +p ti +mix ed +t shirts +mer ci +b tob +women instem +genealo gy +pre ven +l our +cra dle +gi use +Ð ¾ +chron o +fair ness +chocol ate +tor y +as da +pre scott +stret ched +al man +u il +re charge +in tre +ob st +hosp ital +hay ward +teneri fe +fried man +vap ing +confe ssions +ye ah +bal li +luck now +cor pse +sculp tor +amp ton +t pp +indic ates +sur plus +tru man +ðĿ Ļ +sin ha +in vo +sovere ign +ke v +establi shing +engra ved +assu ming +ðŁı ģ +sou za +fab i +ton ed +oun ge +del oit +dow ney +no ble +om or +car tridge +ðŁı IJ +u hur +hol loway +succe sses +r sa +âĦ ¢ +ma zz +tw d +disc ourse +. < +y at +satis fy +com pri +ठ¹ +graph ite +disser tation +ar ter +í Ķ +b ally +zom bi +ly ons +a ic +u bc +pra da +e il +da x +cla i +grand daughter +extravag anza +chall enge +ðŁ¤ ŀ +po ver +primar ily +dad dy +man a +bi kers +inqui ries +da un +fel ine +gener ative +he f +benef iting +lind sey +pol ka +demonstr ated +al le +rand y +o su +low key +weir dest +red bull +our y +n ous +wood stock +cre denti +nic er +g ado +aly ss +ap h +prepa redness +station ary +incorpor ated +dy er +sarato ga +cele sti +: " +antibio tics +or gs +inde fin +ap ron +и Ð +fif teen +no f +ðŁĶ Ŀ +ph x +te ga +m z +organiz ational +on air +band ung +pleas ures +mor i +secre tari +rac coon +ca shi +pil ates +k on +geof frey +la o +kam p +depart ments +back packing +an am +à « +crack down +aun ty +on do +li zzie +ph ers +cu n +ðŁĩ ± +k pop +pu t +inten tional +connol ly +bar clays +hs fb +swin don +u ku +s ally +a int +âľ ħ +pen ang +up lifting +epile psy +inter ro +bun gal +go ku +blue berries +ठ¦ +u ssia +sil ky +mou red +i stic +bri efs +me ats +go b +ch aser +state wide +pra sad +gl itch +ar in +ban ff +memb er +ðŁĺŃ âĿ¤ï¸ı +lo ving +hall a +ภ¡ +smo kers +yak u +scicom m +physi o +sw ol +lem ons +gel ato +ch ool +capit als +ki stan +ti ghts +spi kes +trav ellers +ik lan +commissi oning +ar ine +emabiggest fans +empha sis +front line +pad dock +destruc tive +ba ha +l inger +je wish +shet land +mc gin +mon key +ko z +s one +raj ini +te h +y en +c vs +masqu er +gir ly +we sle +was nt +bro dy +termin ator +gil le +mag gi +bir die +jeopar dy +cu bic +vm ware +intric ate +an up +to pia +east on +sab res +investig ates +bu sting +bil ingual +valent ino +in format +fer re +advent ur +hydr ate +for sy +az iz +san to +e de +whist ler +continu ously +d ham +un used +ji had +addic tive +vi dy +do b +i do +fi ed +ni versary +n one +fu er +ðŁĺį ðŁĺĺ +coven ant +prin table +immac ulate +o em +cl t +serv ants +consu med +un released +sc um +pack aged +me re +ìĦ¸ë ¸ +to by +ta f +spo ons +me al +f ball +fair field +jan et +silver stone +dart mouth +follow me +voy ager +kom bat +anni ver +ene w +mag dal +ho ve +sa th +grizz ly +car di +gart ner +sand y +kan ye +post ure +po ign +im pulse +radio logy +horiz ons +si am +aish war += => +no che +tr is +el yn +com me +du i +ce c +councill ors +cudd ling +creep ing +loc ke +manag es +trans ferred +ne cks +di er +dan o +v ick +lun ches +d he +en sures +cri ss +ul ster +bann on +cont enders +sp am +sweet ness +med al +hon duras +arc tic +ultra sound +in fr +disco vers +ei ffel +ca sters +ru ben +du st +awe ed +atri um +lest we +se ared +ðŁĵº : +ty ne +ex changes +little mix +l le +astron auts +hersh ey +work day +kno b +so v +re signs +today show +der man +an th +af c +ta ster +sw oo +sa eed +per ing +narrow ly +rn li +best buy +panas onic +obst acle +farmer s +ðŁİ Ļ +pa wan +ki est +ang ers +absur d +oh my +sin o +pist achi +sp ice +giu li +prime time +ko w +k ens +ex agger +! ?! +u ba +midd les +ju dd +e jec +slam med +pen sions +of a +re create +b hp +xx l +liver pool +thre sh +pur ity +ni eu +hol ics +wr ath +ra do +gli o +am ma +dile mma +cr u +lets go +.... @ +âĿ ĵ +sugge sting +tru mps +hor us +f v +ic om +refer ring +predic tive +tar ts +ge tte +so ck +glo ssy +pin ky +al ec +thy me +ou ra +thero ad +pe tr +cr am +p fi +dv n +me ier +incen tives +tun nels +mobi l +rec ap +extra s +upri ght +rev amp +per severance +, - +ot p +mir ror +ar wx +ger ry +ma her +g or +hom epage +am is +ag ra +made le +best friend +sirius xm +bun dles +admir ing +t dsb +ðŁį ģ +ch as +slow ing +ro h +wall papers +âĢ¦ / +tek ken +gang s +tal a +lind say +shou l +line backer +tool kit +ur anium +caly p +ab rams +mat thi +ðŁı ¿ +hon ourable +da yo +ver sail +tan k +st c +fr itz +spl end +pat ag +anno yed +on day +devast ated +chattanoo ga +national ism +mas sey +jen n +tail or +dev gn +org ans +zu cchini +on fox +sat ire +wex ford +dis grace +no to +vol ta +âĿ¤ï¸ıâĿ¤ï¸ı âĿ¤ï¸ıâĿ¤ï¸ı +à ¶ +home owners +poin ter +m cr +au sten +day sto +mo ons +pal ma +gra zing +e so +influen cers +shahid kapoor +compli ant +measure ments +develop s +y d +par l +p vt +rand olph +tor tured +ger ald +eli as +deepi kap +war mup +hick ory +g ap +co ffin +am our +re neg +moun ting +seven s +ig le +hi er +dec ad +tri ght +esc apes +wer ner +t fl +ful filled +ni ger +sour dough +re aper +choo ses +spin ner +week nd +fil tered +sh uk +kat i +old ham +open source +kh anna +at elier +conne c +opho bic +gla s +complic ations +ar son +counc ils +sm ol +as sy +lur king +ling ui +han ks +e in +Ù ħ +ru gs +n guyen +nou veau +men ace +le v +alad din +ru ining +round about +k m +con or +shoo ps +may day +traum atic +prab has +ka iser +k ita +rou ter +pe dro +re tar +stun ner +spani sh +distur bed +acade my +e learning +wit ty +sen g +fer al +av y +sta b +ke aton +ur du +ko to +hu i +coo ke +ari an +the personal +u ma +se ap +a sting +rhetor ic +hand writing +munici pality +consor tium +ðŁIJ Ł +glasgo w +ra ya +eli za +polym er +bro th +prac ti +correspon dent +addic ts +gay le +ail ing +o fe +p li +hear tw +st itch +sight ings +prie sts +sam o +slo th +good wood +roc co +sab c +summ it +l ace +pres ley +itt en +cin cy +thepersonal network +s week +pe gas +af con +regi stry +ci m +le th +dic ap +cand ice +flu ent +sm ack +pede stri +al oud +car ac +priyan kach +p gh +ir ons +dol ce +lat via +dece ased +thero ck +cla p +cen e +fo am +morris sey +gre t +essenti ally +com cast +be agle +argu es +ing ed +- âĢ¦ +sa g +ha san +ðŁĻ Ĩ +ðŁį ° +nh ra +kann ada +indic ators +on er +bri xton +at as +screen play +sor ority +sha heed +he em +class mates +tain ment +es i +breast cancer +zucker berg +aur or +en cia +ref ers +kae per +vor tex +com part +lym ph +photograph ing +ste ff +rest ling +par sley +mom ento +th man +lac king +du tt +ocu lus +fin o +fren zy +ra sc +der n +dis missed +noo k +met gala +sh ill +rapha el +maver icks +exhib its +eag erly +c pa +amen ities +. âłĢ +exo dus +ern st +lit a +deal t +womens march +i ain +score board +campe ones +c en +ti ki +garri son +fidel ity +bra g +road map +psy chop +lo e +ble u +ðŁijĬ ðŁı¼ +sau vi +spr inger +temp tation +ru dolph +ac ura +wic z +parach ute +stro l +len ny +zi k +dom s +nb af +al pac +vivi an +ro ve +pre et +perpe tu +sna ke +air soft +infl atable +prin ces +ati e +ffe y +pati ent +m ire +chel le +sl ack +groo vy +# : +up loading +!!!!!!!! !!!!!!!! +siem ens +provi sion +v fx +need y +f ats +to poli +bhu tto +sa thletics +alu ms +t winning +south western +adop ting +last night +man ne +la ga +tw ell +ac ia +-- -- +eye wear +hur ley +fle e +sa ch +pe cker +cost ly +is k +cr ates +polic y +ero sion +in go +wer k +ðŁIJ į +torto ise +therap ies +inter net +chihuahu a +ri ps +fre i +ed or +tai ji +t fc +do d +demp sey +christ in +chen g +hi ps +gra eme +com passionate +cavali ers +histor ic +soul ful +crimin al +ja c +vin ci +expi red +sur at +turi smo +k ona +se aweed +ber ts +le ica +expre ssing +a al +wor t +break fast +her ring +am used +rhu barb +mar tian +cospla yer +y ash +stri al +ra ul +refer ral +dw ts +j w +ad ler +cur tains +gu r +val ence +tyr one +sw fc +coach ed +re born +diabe tic +cho ke +nor folk +investig ative +ðŁĴ¯ ðŁĴ¯ +z id +v mas +phi e +objec tives +âľ ĭ +over due +di vers +mat su +ðŁİŁ ï¸ı +casu alties +ภ§ +al k +stand ardi +re alist +arti facts +pand or +ke x +in vin +( !) +ine y +par aly +mr t +fay e +the voice +on ga +de ed +skin ner +az wx +speci men +priyankach opra +nu evo +bar kley +toulou se +resu mes +football ers +cit i +fe tch +è re +lestwe forget +ðŁĻ ĭ +ch unk +dri fting +manipul ation +equ als +pu tt +ky ungsoo +âĿ¤ï¸ı # +ela stic +par ano +fo y +do ping +cin cy +ss ler +interrup ted +al ay +ado res +ame thy +con voy +ãĢ ı +Ĭ ãģ +black list +gener als +sa chin +bru shed +oun ces +non stop +illi ams +bt sarmy +u av +ru ff +bur ma +bi k +defen ce +schul tz +bo asts +lonel iness +go re +trans forms +alum na +@ @ +ra ppers +ne hru +car o +himalay an +wearab les +ge h +pepper mint +re development +flam ingo +cos by +big baldhead +ag ri +bare foot +sco pes +re gram +gh ana +ðŁİ « +i heart +sa die +carri e +microbi al +ku ala +sk ater +quer que +âĻ © +gen res +reas oning +ch ased +as o +sli pped +en can +vam os +ker s +ad verse +mo il +commod ities +with you +sil ent +hy pe +an de +am ination +whi spe +lit z +âļ½ï¸ı âļ½ï¸ı +ri ff +pp y +lam bs +gan esh +ab sent +regu lator +marse ille +en roll +par cel +wa p +by rd +ðŁĩ Ń +tu ber +country music +par l +contro llers +responsi bilities +we y +ch ate +montene gro +chic o +mil an +l ms +tra inees +appropri ately +un certain +popp ies +ed sheeran +nutr itious +gar o +deut sch +awe some +ãĥ ¼ +comfor tably +land marks +et i +re usable +daniel le +ro sal +co les +just ic +c cs +f anny +ni m +mc u +clin ch +at ene +mer ge +im db +ang lo +uc cino +pan ini +an not +bur berry +feat ure +predic ting +fashioni sta +s ask +imag inary +mm o +south sudan +spe ar +hu bble +jo inthe +coyo tes +sli go +ko dak +sit com +polaro id +roo ted +corru p +ðŁĻĮ ðŁĻĮ +bris ban +at z +ah l +re my +tal ent +aval on +ra da +pau line +locom otive +go ons +ne mo +maser ati +ic u +stu tt +histor ically +sm b +pres by +avo id +so oners +rhine stone +w ad +ri sing +tro t +mo des +reg ent +optimi ze +re ece +sm u +ver ti +newyork city +cor tez +ra c +in case +sin c +fiel ding +e tta +tiff any +al monds +sad dle +k rat +mat ter +g low +star ving +gl o +cra ppy +sl ur +st d +monit ors +recei pt +maymay entrata +mc il +un is +rain bows +cal dwell +pacqui ao +j op +a fe +hoo k +es sen +wiz ard +medi an +fla ws +com s +âĿ Ħ +ing h +ha ynes +anton io +tem plates +ou ter +na w +cardi gan +bel grade +ðŁĴ ī +hom o +a ise +ro pes +no ve +what you +tri gge +concep tion +ad ukone +na di +fri ars +sw er +adju sted +hot line +san ity +kau r +down loading +c gi +ten or +eth nic +app alach +ภ¸ +pa g +gol ds +on set +investig ator +car tel +peace fully +jarre tt +cat alan +poli o +n um +fru stration +dhar ma +my life +âľĮ ðŁı» +aber deen +mu sa +bin der +spark ly +fle eing +instin ct +co ping +domin ance +ill ers +er a +u conn +lo oms +living ston +gal i +he s +c ma +bel a +se ley +mon k +la ch +mar x + ´ +m erica +woman in +es sex +ra ina +jim i +nep tune +z ack +chine se +mart ins +chand elier +her n +with us +ear l +asph alt +modu les +st p +ul la +psychi atric +mile age +captiv ating +si der +men to +mor t +tran ce +tal bot +ab by +ì ĥ +âľĮ ðŁı¼ +j ak +daw n +turn up +scre wed +fe ds +blue print +ðŁĴĸ ðŁĴĸ +har sh +er os +insom nia +ban kers +ta emin +mis conduct +hu mber +gi di +edu ardo +con a +musc ular +consu ming +ra sh +don nie +di pped +col lie +samu el +melt down +ðŁĺįðŁĺį ðŁĺį +me z +exam ining +schwar tz +pri stine +ðŁIJ Ŀ +ve it +ful filling +an esthe +gue sses +dra ft +som me +soli d +pati onal +ho ped +evolu tionary +all er +enter tained +sli ps +lud wig +conclu des +sen sible +bon net +cra ze +tra s +haz ards +const antine +ed ics +star trek +to c +occu pational +in cheon +deepikap adukone +pizz as +new comer +de part +oppre ssion +ebon y +foss ils +tro jan +el en +ste aks +k hou +positi oning +ug by +red cross +ak h +dol ce +us mnt +pp en +dil ig +ma vs +call er +cost ello +⼠Ħ +dy n +thing s +rhin os +a xi +sar kar +con vocation +att ers +ss ss +fun gus +eu gen +russ o +squ at +w sb +eli on +william sburg +s off +defici ency +be arer +o kin +key stone +t wain +cal ming +break able +wa res +horser acing +com bs +bun ting +u it +t land +ðŁĴĻðŁĴĻ ðŁĴĻ +ga stron +sab ot +ick ers +commissi oners +sen ate +ii ot +ath ena +nit rogen +an tony +ero tic +di alo +mis sou +hypo cr +âľ Ī +kaeper nick +can v +d roo +clevel and +o sh +mon sta +stefan o +^ ) +sh ul +po ison +ha e +commerci als +ma ul +nit ro +co worker +alo e +vap or +t ents +russi an +qu id +question able +mid get +po ker +girl friends +sin the +erit rea +ten ure +depos its +buc keyes +spot ter +theod ore +trin ity +joaqu in +u cci +follow the +caf c +mp a +ðŁIJ » +plo tting +dom ino +ta ek +sion ally +dicap rio +pa p +car mel +ig er +bt cc +beth le +www bigbaldhead +foo die +bagh dad +mason ry +off ended +à · +ภģ +sc ro +vers es +ori ent +ar ches +pi yu +know your +gre e +ta kers +gu ard +dish on +bucket list +bha fc +war dly +ðŁİīðŁİ Ĭ +leigh ton +pe w +stra y +assaul ted +in hal +ly fe +amar keting +l x +kat z +ubun tu +me o +carto onist +turno ver +mi z +dis like +mul len +mo f +bl and +hi des +emer ges +chori zo +truste e +ma hog +lan sing +paralym pic +fa int +fa una +ch al +sn ar +cat h +bent on +cast illo +sli ppery +apric ot +oec d +bar o +l z +he ming +clow ns +co workers +peru vian +commu ters +y ell +ðŁļ ´ +under ing +v j +tt p +fli pk +w ana +soc ent +Ĥâĸ Ĥâĸ +ठĤ +oo sa +jag ger +di sm +e less +d ham +cali f +a official +ec lip +harro gate +gra pp +com rade +n tr +concentr ate +thi ghs +bit coin +bel arus +ë ĵ +end uring +now watching +industri al +pi p +ar on +ar at + ® +whit by +oooo ooo +sa ree +tic als +mis leading +yo on +year s +sle igh +roman ian +sciss ors +vam pires +ac up +ab ba +th weeksary +cent ri +fl ye +u o +c bi +bu ena +sin d +mar ino +bur r +re building +ठ² +anniver saire +ac ca +ðŁĴĢ ðŁĴĢ +gett ing +tu lips +wolf pack +âľį ï¸ı +more than +ta kin +ðŁ¤ĺ ðŁı» +u be +mon ic +dou bts +mo wer +co balt +don ne +specul ation +argu ably +kak u +htt ps +prosecu tion +din ah +stam atic +disclo sed +bever ly +fl wx +cra bs +extraordin aire +war mest +imper i +o logists +trac es +par c +lake side +am r +ter i +hour ly +domin ation +ar row +shrews bury +ance stry +wr angler +trigge red +pen sac +roo ster +survi ves +a on +bo ko +val or +love is +la g +pe y +fo cal +out laws +bl anc +artic ho +wit s +marsh all +die go +support small +u ca +sa h +je et +syn ago +gover ning +ðŁĴ ¬ +sal ads +cre ate +miri am +cen sored +ami de +no u +z eta +allegi ance +* ) +bl m +ric an +pa stors +oly mpus +blo c +whir l +star ry +pr one +y k +p ne +congratul ating +be v +so ber +love island +sa ir +an ing +tutor ials +q e +lun d +in ist +cle ver +taxpay er +ali z +wren ch +dd ling +cap ri +h pa +ðŁı» âĢįâĻĤï¸ı +na j +o j +futuri stic +jelly fish +ðŁĶ¥ðŁĶ¥ ðŁĶ¥ðŁĶ¥ +cel ery +plan k +fil a +ne me +un healthy +lec tions +ðŁ§ ¡ +rit chie +n ws +mi kha +wonder woman +âĢ İ +hip stamatic +ka g +ðŁĴľðŁĴľ ðŁĴľ +poul try +mo w +wor ds +lo ff +ðŁ¤£ ðŁ¤£ +relat able +re mixes +keny atta +ke m +re signed +fo d +stra igh +j lo +hu tch +box ers +colle en +mag s +instruc tional +ko l +attrac ts +pra g +account ant +go ggles +br u +th ole +mar row +leu ke +oc to +pon ds +bubb ly +he ist +ìĹ ij +im p +a har +ha unt +hall mark +psy ch +kkkk kkkk +col umb +jump suit +cost co +si delines +ag gies +over turned +ni b +key chain +fu k +f af +mi am +assist ants +cy cled +ri der +dam mit +red wings +mag es +kin s +ì Ĥ +ho d +son t +carol ine +" ' +cu le +bra id +fel ony +ar ities +ruther ford +depic tion +isab elle +ro ach +k day +fifth harmony +em y +li gam +bari sta +albu querque +gro ss +ðŁį º +oo ks +ðŁij ¼ +dun can +try in +jag s +g ould +li tho +âģ £ +а Ð +sam my +tun g +cas ser +apo lo +aaaa a +man g +as ics +sh en +p ye +tur bul +ss p +saint sfc +on lin +n anny +he ster +do z +ภĶ +th read +ren ts +kh and +ðŁĴª ðŁı½ +un conditional +rob son +car re +ph on +sacrific ed + £ +auto s +par ker +oc a +log in +kee gan +hard cover +dough nuts +ðŁĮ İ +spit fire +refresh ments +saskat oon +commod ore +j f +rub ber +halam adrid +child care +stra da +io m +ri k +dak ar +ther mom +cro pped +gar u +ali k +ven i +i ft +si ka +ritu als +z ul +e ch + © +su dan +l land +i me +do cker +ì ¤ +fe ared +fa o +wal ter +no g +mutu als +l h +ali gn +mon ia +concep tart +ðŁĻı ðŁı¼ +sco e +compet ence +sw ine +ly me +laun ch +green er +abstract art +inqu is +gran ada +ga elic +flu ff +d backs +grave yard +ba be +acade mic +adventur ous +joh ann +~ ! +bi bi +| # +pl ings +gett y +as b +âĿ¤ï¸ı @ +staf f +religi ons +bang or +world bookday +me gh +de vin +ash ore +meri dian +gi thub +qui z +all stars +be stest +ir resi +ack er +do te +war rington +pol ly +newor leans +cr ou +wi gs +che y +smithson ian +la sag +de tour +bor is +stra ps +mari ah +inten tionally +ko h +ðŁį ¸ +ssi an +mar issa +cor al +episcop al +casu alty +tom o +supply chain +sam p +on go +ro o +cavi ar +p fw +clau dio +buff alo +s ations +mat ty +snap back +l ds +al arms +mat te +âĺ Ķï¸ı +conditi oner +d ors +he x +fi zz +a stri +sus sex +secur ity +qa eda +all star +cocac ola +as one +cl icks +sc ans +mu te +he avier +ðŁİ § +âĺ ŀ +lv l +book boost +youtu be +fla shes +f jor +c su +explo de +do dge +cair n +gonz ales +th ill +pel le +hart ley +renew able +re tin +e stre +costar ica +shipy ard +nc fc +pri ya +a ghan +an ath +plu gin +co rey +re bound +or u +kat rin +hor mone +gi m +mahin dra +s sus +park land +har per +fanta stic +infer no +ep ilo +wrest ling +fe ct +c it +ac oun +to ssed +monu mental +char tered +bu st +pe tra +âĮ ļ +wildflower hour +sweat ers +* . +bl er +ate ch +go wan +demo graphic +bra l +suici de +renov ations +vu el +sin ister +ar mani +miso gy +ph arrell +nap s +un iting +crusad ers +cor gi +insu red +than i +no or +g q +d ada +bicy cles +snu ggle +sch an +ten berg +ss al +fe mme +bo il +½ ï¸ı +re ap +occur ring +hus sein +divi d +sto ke +sh alom +na ia +o lic +frustr ating +Ù ĩ +ig s +gro ver +scen arios +n ds +bru tality +med alli +bu on +sas s +skate boarding +ony x +lor ry +ny u +gau tam +mm ings +gu g +end i +lo thian +comm ando +chal k +ph ora +asse ssing +ti gh +crun chy +ad ay +is l +ci ara +pilgri ms +kam al +p to +brit anni +t ani +sm c +l ure +app store +ab y +golf ing +cl c +fa u +an as +shu tting +regul ated +carn age +scow boys +all enge +c ma +humbold t +rel le +ku mb +her i +refin ery +sound check +d wayne +bos nia +i sp +the alth +anni v +relev ance +my a +bag gage +dre ad +s bc +th ed +bu h +hi jab +lo id +ke w +c te +respec t +lovel ies +cu bes +celebr ate +dir t +sav ers +_ , +gar ment +pulit zer +mas jid +beat port +al arts +encry ption +s ner +ple ads +found ry +sym metry +ru mi +birth place +scallo ps +supp le +pivo tal +t ati +no de +so d +pro xim +tr ics +col dest +bren t +mand u +cla ir +e ach +and alu +hi ddleston +ðŁIJ º +mel ts +v ance +pin n +se ments +scre ened +sa chs +o bl +ic ha +âĺĺ ï¸ı +school ers +heal ed +lo gged +ðŁ¤ĺ ðŁı¼ +ic us +bore dom +b ish +b ffs +tal king +sure sh +hoo kem +de on +de fl +ei leen +ðŁį ķ +women intech +ri sotto +rang er +adverti se +ภģภ+tel ly +la go +dart moor +d ong +sk ates +lo go +un ner +mail box +ma sala +lo oooo +amethy st +che wing +c bb +australi ans +rc mp +game art +# ... +kor n +extre mism +fruit ful +anci ent +pu bg +pol ite +wh it +mur als +m gr +line man +dav ao +ste ms +ten nis +av age +tu pac +gigan tic +hs bc +auto biography +up the +ี à¹Ī +re gal +fig uring +ku l +mis sy +hoo p +gra s +for ums +back lash +abduc ted +p nw +min ic +bu tt +bott oms +at on +ven g +ðŁĮ ı +del aney +prab hu +fan club +over haul +health ye +sy no +aa f +ren amed +kim i +un cle +man city +se u +qu anti +este em +um in +en zo +mel vin +under go +j har +far ah +coast ers +humph rey +mh z +children s +^ . +d hi +disrup tive +integr ating +r nb +over sized +a ide +ne au +docu mentation +ðŁijĢ ðŁijĢ +pal o +hear th +ri yad +pun ctu +abc news +secu res +boy band +bir ch +ju co +tra ff +legislat ors +bay a +ãĤ ¯ +no ises +collec ts +s warm +k ner +bi shops +stur geon +snapp ing +mo l +fre aky +chair person +tro p +lyn ch +car cin +art sy +e sto +cha i +fl ur +inv ali +sau sages +im el +j or +fun fact +wit ter +puni shed +ac ons +h ya +re versi +em c +dif fu +z x +sp aw +cla d +d mit +hol land +fre sco +pay roll +ab undant +stu ffing +mor o +c ny +boy cott +wend y +ele ven +pro voc +pil ot +tr x +be ad +climate action +ri on +assi e +ì ĸ +o sm +islam ic +ho ar +good reads +al ici +afterno ons +spoke sman +jo lie +it as +masc ara +âĻ© âĻ« +pre vail +beetro ot +lu jah +k li +dod ger + » +ru le +l n +scre am +ho bart +col bert +r tc +er m +pat ro +quo ting +s live +que st +non fiction +semin ary +prosecu tors +ve st +express way +g ge +nau tical +et f +ðŁİīðŁİ Ĭ +dur ation +cha ired +the film +fab io +she h +can o +ðŁĴª ðŁı» +with draw +! :) +cor pus +phen om +yel p +la wn +ent om +snapp er +but te +pin ball +pro xy +libr e +alle vi +n ada +gabri el +fo wl +eure ka +daph ne +tu nes +pun ched +wh ore +jo g +ren tial +man ners +o pe +wh ufc +gu th +revol t +sne aker +philharmon ic +ho ste +sovereign ty +ðŁĻıðŁĻı ðŁĻı +fish ing +sci art +fe ta +i pp +dump ing +kel own +gir i +dig its +sal u +san jay +twee ters +sp as +col chester +sc ab +ma dd +๠Ħภ+Ä ĩ +ged don +march for +do p +maure en +un plugged +di do +fashion blogger +up a +mex ic +tar y +pol ye +jame son +v t +grin der +mad dy +consult ancy +¬ ë +leagueof legends +ac cents +um ni +jane iro +tu ss +h ens +ampli fier +to shi +pret tier +pre vents +new town +red wood +vant age +ball ard +ar tof +a she +a sion +lac ey +ap at +gro ve +ภĦ +rw and +real tors +tra itor +bed ding +ö r +zi on +fla shing +cam pan +boom er +secretari at +ab ol +liti gation +cont amination +se dly +shred ded +in for +do herty +bench mark +ro che +skate board +sho vel +i zz +to pper +o ster +laby rin +autu m +k ong +hum mus +vi z +tech news +kla us +am using +socialmedi amarketing +i des +cast ell +ste e +underestim ate +cal ab +pa ign +b illing +unanim ously +g mb +fly fishing +hath away +commerci al +colour ing +skul ls +pivo t +te p +tb c +motor way +x press +construc tive +pu k +under lying +kir sten +mani ac +cha o +se ma +chiff on +ðŁijĮ ðŁı» +ver ona +kom o +stan doff +wi ped +c ated +bla ir +wor kin +m sc +bethle hem +swi pe +unexpe c +pe es +pe tri +orig ami +ðŁij ħ +mex ico +flav or +ru dd +cannab is +mar u +ri ddle +wor shi +sil on +sch at +ap se +tang er +bi ous +e er +questi oned +o zar +dan k +angle sey +char an +bak u +compe ten +re pri +bat ter +sa xon +cal ves +leng ths +$ $$ +âŀ ¡ï¸ı +immer sion +ga unt +car ry +cy to +b anda +shu tt +experi ence +el gin +mous se +ta z +ê µ +in correct +en z +b ham +mor on +so ver +ar un +ti pped +la ble +de arly +bau tista +í Ļ +mor tal +woo p +dt la +sho cks +dav os +ðŁĵ Ŀ +swim wear +her man +ðŁijĩ ðŁijĩ +z ir +neglec ted +grac ed +campu ses +av s +ar ora +swach hb +live pd +ac cra +enqui ries +shoo ters +kur t +vancou ver +brad ley +gar da +g ü +ol la +attrac ting +up ton +ne win +lu mia +furn ace +ev ers +e on +sw a +roo kies +a oc +v ss +bris ket +tor ch +yo da +heart land +tac o +ph ony +food bank +ab bey +bab ylon +u y +gre ate +expre sses +d andy +sc apes +survi vor +ron d +e ci +ha vin +ab el +chil dish +tor que +wav y +ur self +kanye west +year of +ale stine +o brien +al fon +sk ag +kore an +anchor age +val eri +de w +ðŁİ ¨ +land slide +car ole +christ en +go phers +af i +priyan ka +q q +power of +it te +pc so +tw ol +pr y +intellec tu +guer rero +pi les +wish list +w ren +time table +ë ı +prodi gy +gibb ons +. / +ne ur +anz ac +mur ray +vie st +pla ster +la ir +art gallery +inter continental +g br +bell ator +nam joon +mam mals +am el +y aw +saras ota +cam ar +bud ding +sum mari +aco sta +la sh +ey ou +post graduate +instruc tors +ti g +const ant +were wolf +ic os +cla s +glen n +bud ge +ðŁĻ Ĥ +er ta +sta ins +persecu tion +cumb ri +o ch +syner gy +hu ang +scand in +mid terms +comment ator +regar ded +perpe tual +bo iling +al p +lan ge +sch le +fac eli +twee ta +ri dden +ok toberfest +charlotte sville +ik lan +jo u +ch atham +b sc +ðŁį ¦ +stra uss +mel low +xx xx +happy hour +re actor +ww er +distr action +at orial +ðŁĴª ðŁı¼ +twin peaks +fay ette +a or +ko k +bro om +sy fy +ou se +am ag +Ø · +ubis oft +lu lu +hall mark +stu art +it ya +si deline +venge ance +re lu +sex ism +boun cing +un ites +gu stav +te ssa +stu mp +pro clamation +ima x +divid end +col by +ðŁį İ +play wright +un safe +co smo +ðŁĩ²ðŁĩ ½ +cup board +constitu ents +ang lia +ram page +ðŁĺįðŁĺį ðŁĺįðŁĺįðŁĺį +than ked +take aways +shro ff +de bat +kh ur +conduc ts +format s +à © +port age +graph ers +u ten +pre m +mo ines +condem ns +s ous +l ps +f cs +deal ership +leuke mia +bure au +ski d +guardi ola +ca ster +thir d +avoi ded +en cyclo +c sr +vi xx +analy zing +she ar +dulu th +shap iro +chan ting +stre sses +as be +mil itia +ãĥ ª +col lin +arsen e +sure sh +teach ings +yi xing +sh ill +nu des +sv u +clear water +war ped +pro life +artist son +it u +versail les +galax y +ax el +spring st +cal a +hu hu +sc u +commit ments +exe ter +poign ant +mo tion +conserv atory +row dy +rec alled +mu sk +emb elli +so the +âĺ Ģ +sto pper +sch ild +to pe +el mo +zi el +j om +barn sley +snow den +on tour +jour ney +hills borough +par ole +w ts +mo ving +ag ility +tiv o +ff ers +kindle unlimited +g wen +ann an +ah mad +tex tured +hepat itis +dra m +insi ders +tis sues +ãĥ Ħ +fc barcelona +cr atic +na acp +pe can +f gm +custom ize +concer t +g sm +pe g +p one +justin trudeau +super cars +happy holidays +bu lar +ado x +lap tops +digital health +destin ation +gradu ally +áĥ ¦ +popp y +ss l +inhi bit +star light +of fro +glo omy +x per +hal der +im plants +le to +hass el +a as +un told +en ci +liber ia +or an +con tests +il ah +sma g +sc out +mari anne +cr yo +schedu ling +lo s +kan e +stutt gart +ne se +law rence +da in +pho tom +car ou +ภ£ +g wy +national dogday +roa sting +band camp +kentu cky +stret ches +ke rel +ca she +ãĤ ¸ +sta x +tran si +dog gie +at ric +hal le +ci vic +brow ning +lein ster +cat day +high land +joy ous +in cumb +or lando +ro mo +col ton +del ta +car ab +ro tc +aster oid +goose bumps +mo logy +yo ko +an ds +tomor rows +red carpet +sm p +ca sio +ðŁ¤£ðŁ¤£ ðŁ¤£ +se au +rejec tion +rot ating +bi partisan +th un +mat i +bon i +ol l +ener gye +do it +l j +mother hood +lou ise +neck laces +el ite +ni x +l cs +en v +gl u +le sh +cran k +su sie +m clau +so tu +crow ley +rat ri +use d +bre ton +alfre do +ye o +travel pics +ti pp +elli son +sax ophone +me red +heu ghan +ta ine +f es +vi ro +suppo sedly +i as +dige stive +y le +li zzy +wildlife photography +bri anna +west field +ra ined +am her +ðŁĺĦ ðŁĺĦ +distribu te +bott om +pre serving +oil and +craf ty +de scen +col ling +shakespeare sunday +r wc +ang led +ci an +t ations +mon tage +me yers +france sca +ðŁĮ · +wi ggins +san ford +volunte er +car ra +bar k +vari ed +pl in +am u +kap il +rock ers +qu ind +br ane +in mate +ent al +impro vis +michi gan +re tweeting +progre ssing +mercedes benz +smo ker +physi ology +dor ado +watt pad +h wa +sr bachchan +w ga +vol atility +hi re +ac ap +wn ba +hein z +stit ches +kidnapp ing +bur ys +lim b +f itters +thumb nail +ton e +mir and +desi rable +ad dison +tar an +tamil nadu +spec tator +soci ology +amit shah +remo tely +âĻ ¦ +ham id +r ds +g lee +smooth ly +sch ro +er c +lali ga +he als +us f +ni shi +d hu +un il +h le +tro mb +bhu tan +pilip inas +se ung +whit man +te y +min ce +snow boarding +re au +k ker +av o +zach ary +ran veer +ti k +gover n +qu al +beck y +anthropo logy +att en +grocer ies +de bit +war p +sil icon +hawa ii +ðŁĴ ħ +pomegran ate +pe er +orang es +people schoice +end ure +ðŁĴĽ ðŁĴĽ +ãĤ¹ ãĥ +ac ial +a haha +stu k +imper ial +bl ond +pow der +kno ts +vin ce +wood lands +den a +watch in +mat cha +ma hat +galax ies +middles brough +k ö +stre e +resc ues +wal do +lero y +desp ic +real ities +tm nt +ha q +un o +pe c +bolly wood +blin ds +design thinking +he ms +and hra +ab sen +fan s +ste ch +shire hour +bla ine +shak ti +pu rely +ðŁı ı +tra fal +ke ynes +gr ate +to bias +spon taneous +satur ated +caval ry +pri sc +ðŁĺ ij +wh t +pas si +~~ ~ +vir at +patt inson +la o +weir do +sym pathy +ju da +occa sionally +cred ited +stat u +es co +hil ly +esc ape +dischar ge +se er +may nard +sud bury +z lat +or al +we er +encoun tered +sm elling +over sight +ê ¸ +that cher +mack ay +you can +fre ep +freed oms +prophe cy +ho e +ishq ba +dra ke +qu its +pel led +tur k +o vi +wesle yan +new music +leg g +ch eng +h illi +ay y +pan ties +ad versity +ad jac +vaccin ation +ju ke +ga c +exce ed +time sof +sta ining +ep cot +v ital +up ward +bethe sda +apar k +ma hi +camp fire +enchan ting +rha pso +h z +na ver +fa x +vali dation +ac ad +ny r +as ym +coordin ated +depar ted +all ery +var ies +spr ite +chap lin +ss occer +s wat +bre t +relu ct +tunes app +super star +reminis cing +o co +home grown +dough nut +un canny +la pd +thyro id +! âĿ¤ï¸ı +botan ic +bre s +sp ade +i ste +echo es +du lil +bur sting +qui ero +ðŁij İ +loy ola +amuse ment +ha ils +sleep y +burgl ary +âľ ı +ro gue +cot land +mo ors +low er +wic ked +ðŁĶ Ĭ +compet iti +argent ine +yvon ne +karti keyan +ili ary +gat sby +precin ct +six ty +na ji +cam s +practiti oner +ðŁĺ³ ðŁĺ³ +pu ne +neg li +juli en +inv aded +cali br +cla m +duba i +mu k +lan tic +produc t +fe dex +ï¸ı : +eu ra +dari us +s ling +virtual reality +home stead +ðŁı³ï¸ıâĢį ðŁĮĪ +pac ed +in ha +pul mon +la zy +premi ering +ma stered +in he +con gregation +ba jo +sport ing +new jersey +hor ny +lma oo +leng thy +du t +yo gh +swe aring +philosoph ical +pap ua +in ski +know les +dy ke +âĢ ² +to ken +mc guire +ri ot +probab ility +mc con +gro s +su mat +c ite +da a +on da +mad dow +che w +board games +spar ked +re claimed +ad hd +ny se +imwith her +equ inox +boo ths +balsam ic +ha zy +dor chester +ag os +se aw +moder ator +seri ea +ander sen +pilgri m +âŃIJ âŃIJ +itch en +hal li +x ton +nathan iel +mun ition +celesti al +ga f +zo om +mark le +pen thouse +cal e +s fa +bar king +tu cket +em ery +cal orie +li que +ad ar +mc nam +tor tilla +wood pecker +mo town +bad ger +ayr shire +scram ble +dd ay +cra ziest +per rie +cho co +cast e +i ot +wre cked +selec ting +uss r +gra ft +pun t +lab ou +ir st +ba ek +Û Į +su ki +que u +ach at +te ster +aug mented +wc vb +sin ks +ðŁĵ » +ra ke +inter ne +be cause +belle vue +une arth +light en +ðŁĺ £ +turn around +labe led +unemp loyed +twitter kurds +le ia +h ye +great er +ðŁIJ İ +tim ed +i red +e tt +limit ations +cab e +s out +bee ch +anni hil +re trac +yo ona +ang er +den nis +supp lying +di z +" ( +sc ur +gun man +su ho +sauvi gnon +ภ¥ +wi ley +land on +choreo graphy +pre historic +ðŁı ĥ +var gas +assess ments +pinn acle +di i +chamber lain +ì Ī +v p +present ers +deut sche +sun shine +sal utes +r one +bu siest +- .- +motor ists +hemi sphere +al wx +ps p +ow a +den ying +cho c +gu tier +han uk +mus kete +jait ley +se wage +t ame +thin kers +shi m +se quo +pap ar +middle east +k wa +ke g +patag onia +no y +bar ça +take off +he a +à ¬ +n sc +g dc +ðŁij Ī +mou stache +mel ania +thr a +â¬Ĩ ï¸ı +pier ced +ze us +fon ts +ber a +it iner +q atar +contr ary +ire land +i fy +ou los +commun al +fin s +un paid +pa a +ðŁijĩ ðŁı» +ri os +ou p +f iller +cafe teria +à¸ Ń +kas i +cali ber +z ulu +v sco +ts ford +dragon fly +smo kin +pi st +psycho logist +diplom at +we bs +buc cane +à® ¾ +motiv ational +du ne +ba e +c fs +with out +er on +i ac +ate e +pen sion +fra zier +en sis +sk is +par ting +ger y +territ ories +nach os +eni ght +ever lasting +msd honi +tel e +sp un +po di +sab ah +environ mentally +ce ase +beau mont +mar ta +kel vin +ho ff +sun il +n da +co b +sh ale +ree dus +un boxing +u bio +re opened +n all +capsu les +mar r +himalay as +swee ter +ja z +f mr +twee ter +dha ka +na u +de mi +d fs +ta urus +fad ing +it utes +ci p +over flow +jef frey +don ny +car tunesapp +ðŁį ij +prefe cture +danc ed +c pt +ple asing +ital k +earth quakes +ul ation +hi o +ãĢ ĭ +ant an +nutri ent +de ere +selec ts +enrich ment +r iti +tram pol +bl amed +j ia +contribu tors +chesa peake +pi geons +tribun al +mad uro +w su +ilo ve +effici ently +dar cy +war ms +ar ra +ec u +ho wer +strugg led +rajini kanth +ðŁĺ¢ ðŁĺ¢ +hou sing +str at +eli x +disp ro +raf fic +thi erry +na sty +c fb +staf fing +al ma +back ers +hen son +sky walker +reale state +roo s +ness y +chan ce +cair ns +c ci +pe dal +ly ft +cross word +wait er +only in +kru ger +k ir +alej andro +car tier +car rera +re paired +ou at +un clear +un breakable +today in +qu eries +jo dy +gen ital +win ner +to l +kelown a +fascin ated +ãĥ ¬ +sris ri +squ ared +spr ung +negoti ate +priv ately +av en +>> >>> +g ical +gav in +chester field +zu mba +or r +nat alia +impeach ment +mn l +car at +criti que +credi ble +trac y +tan i +musi k +jig saw +gam bia +tol kien +fe u +as per +sav ory +fo xx +f itt +mar lon +l rt +v ell +p br +imprison ed +i om +chu l +wind shield +kay e +ba a +chor d +s art +al gon +minister ial +nat geo +la zio +nor ms +ðŁijį ðŁijį +lic king +fut bol +un sung +dalla scowboys +sh red +distur b +dev ine +be ards +ch f +b day +ro sso +ig or +ay i +si ren +k air +sti les +ro f +mag nets +un cover +mou se +bang ing +si ghted +spe ople +impac t +row land +kir a +environ ment +love the +p sis +mish ra +gl endale +ca jun +o che +de ception +sex ist +stra ws +s ga +buff er +apost le +sp l +pop up +ðŁļ Ĺ +r g +up er +ball in +i dy +occa sional +national park +ðŁı Ĭ +u an +innov ation +ภ« +te aparty +re tte +counter fe +b ha +rec s +ig en +ðŁĮ IJ +humming bird +cu r +ha ven +la zar +pue blo +: : +zi onist +op ath +inver ness +promo ter +carto on +cabine ts +mahog any +surve ying +r ational +feel ing +testi fy +so w +oc on +ภ¢ +ne el +mar is +sol itary +che mo +rad cliffe +sim ons +ros ary +new er +jo die +re tali +pra wn +pad dy +hen ge +k ala +im plant +at y +bren twood +par adox +ene z +re designed +p our +wy d +al de +௠ģ +sol d +biomed ical +๠Ĥ +tt tt +mat teo +ys er +new ton +de bun +ner dy +loo l +wo on +elisa beth +ec c +wh i +ach o +salv age +sal aries +qu ity +navig ating +oph thal +con soles +re built +o pec +ast ers +sho red +set list +kathr yn +rhy mes +re visiting +ash ish +li ft +re post +sole il +âı ± +weal th +sa at +we c +king james +flipk art +field work +se gu +mo dal +bu b +are rs +ðŁį Ĵ +clo oney +pad dington +necess ity +guth rie +pen te +li mo +jo sie +ar tin +en c +l hs +betra yal +info graphics +i er +mo a +hear ings +bon jour +sym bolic +ag ro +wed ges +krist ina +wild flower +athle tic +photograph y +pe sh +ca hill +chi lean +gou l +fi oren +ðŁij ¶ +z il +sk im +bad oo +deli a +tre ble +n cc +ðŁĩ¦ ðŁĩ +a house +bul lock +sol itude +ا٠Ĩ +can cers +futureof work +hu tch +water shed +war mongers +sp illed +colom bo +mo th +associ ations +weigh ed +global goals +not just +christ i +tor g +swe ating +man eu +clu sters +âĢ¼ï¸ı âĢ¼ï¸ı +ta ped +ul y +tru sting +yu suf +te in +ra b +, ,,, +sin ai +audi ble +explic it +cro wns +sch iz +at least +ðŁĹ £ +de bra +je suit +ene gger +z hen +one sie +i it +ss f +gur gaon +chak ra +bear cats +k ran +k awa +reque sting +han over +g end +sor os +mer cy +lovel y +do omed +tim my +ku z +ul l +ab ram +sa ison +ãĥ « +clean ers +re mo +circu its +bar red +o th +mo ist +madele ine +gall o +u j +per mits +hea viest +car ols +az te +gior gio +flo ats +decl aring +us rc +min at +craf ts +pri ma +conven i +nickelo deon +danc ing +ceremon ial +blo gg +tw p +anglic an +she k +k nick +( (( +hubb ard +harve y +hit man +fen g +we some +for za +s word +op us +bro m +gi bility +z al +m unch +dance hall +gre edy +hd mi +re birth +ðŁĺĭ ðŁĺĭ +s world +figur ine +com post +k f +engra ving +gior no +st ana +k man +ham ster +compos ers +aj e +func tionality +pol k +is ons +air planes +te se +hor rors +musc at +gi ven +sp ence +ðŁĩ¸ ðŁĩ +eli ot +ach illes +fre ck +crypto currencies +sou ther +hal o +bor neo +polit ic +hahahaha h +up state +si ena +obsc ure +hau sen +lloy d +happy friday +motor bike +bon a +americ as +hol s +- ( +spor ty +un aware +reven ues +christop her +bank sy +av an +ev apor +com press +eyel iner +to dos +buff y +renewable energy +ly rical +ar chan +rapi st +fair trade +lma ooo +beat z +pro active +la pse +ir ical +revers al +po de +mcin tyre +mac au +ãĥ ķãĤ +nash grier +f sa +g all +çĶ Ł +perpe tr +il ya +configur ation +% ; +str ange +rac i +ภĩ +pic kups +kov sky +mam mal +w ps +g able +compar ative +z h +save our +da vey +on etsy +mu ssels +mis er +cri stina +electr on +cra ve +lo ren +precipit ation +m z +ðŁį « +vin cen +snow board +no ida +ah n +marin ated +g tr +town hall +min is +bethe l +adv an +su ra +shi el +fur ry +ðŁĺĤðŁĺĤðŁĺĤðŁĺĤ ðŁĺĤðŁĺĤ +lyn d +so il +sc ence +sen eca +shar jah +dick ens +credenti als +av ar +per k +requ iring +pre fer +j ian +de ca +r ach +ing for +del e +be ep +ðŁĴ » +cis ely +hu ddle +green sboro +haw king +ho ax +hang ar +ç ľ +mis o +lo vin +gre ta +ab ad +logi e +at an +snow flake +mahe sh +fear the +al kal +bobb lehead +ba hn +ju dged +fu tu +feli x +ðŁį ĵ +pi ke +der iv +notic es +au er +dis super +or da +wi pes +am ino +stri kers +foo tb +dram as +pun ching +score less +heming way +bi h +bal lad +chat ter +am mo +kle in +fabric ation +kari m +z end +hi sto +vol ta +rock y +marke ter +xtre me +sequ encing +paradig m +cle ats +boom ing +âģł âģł +block ade +promp ts +yogh urt +pur pose +nu r +regu late +nois y +ing rid +bird watching +bar tender +Ù ĥ +wor dof +cha otic +shor ty +el dest +z app +onceupon atime +fl yo +rit os +mike quind +ðŁIJ ´ +regi stering +. ] +ad ol +gg gg +pur ge +kid lit +ar bor +val ves +synago gue +o th +unanim ous +veri fication +dar rell +ãģ Ħ +vander bilt +tape stry +pro sper +did dy +dra fting +de cep +marqu is +st int +michael jackson +pee led +men us +bb b +sc are +ema il +wri gley +it is +f ell +some thin +bar ra +ed gar +di pping +pu ddle +sla de +lear ner +jal en +ðŁ§ IJ +the daily +mikequind azzi +ju x +iq bal +mckin ney +ra iser +ef an +dr one +cat o +pic ket +cro we +l att +uk o +giuse ppe +hin i +synthe si +ponti fex +song writing +to d +swit ches +din ners +h q +gabri elle +pensac ola +cir cle +expo ses +ev s +riyad h +pro men +o ck +sa j +cit ation +brew co +jo si +ep aper +dri f +point less +tang led +cri pp +line ups +fairi es +daz e +mour n +bla dder +sal z +bur undi +book mark +the people +sub sequ +princi pal +sk er +court ney +a oki +rac ers +ad m +mom a +critical role +hou n +shed ding +sa ka +ace ous +mck ay +hus bands + ½ +me da +accu sations +ro sel +nc is +witne ssing +or ama +go ds +hil ton +el man +ÃŃ n +meg ap +cra ven +announ cer +crit eri +sheffiel dissuper +milit ant +consu l +hoo ded +aby ss +b x +ma dam +lo cu +mary am +manic ure +grat is +ac tresses +ros ario +this dayin +king ly +gn ome +cel ine +r ous +he el +lil ac +vish al +ab h +thor ns +s ls +ne al +construc ting +be ren +s lang +ma ins +far ra +sar ko +pai ge +gu iller +l ala +ice berg +nou n +plann ers +u mmm +ou ses +ill ary +ma an +box ing +zi pper +srin agar +migu el +o str +mp o +responsi bly +lan terns +appli ance +x b +gren ade +neglec t +dy sle +ham mock +ne ctar +wit cher +r gv +di ence +ser bian +seed ed +cru z +bi sh +sp he +e q +sky rim +alge bra +phil ately +bungal ow +ge off +y ves +demand ed +consider ations +the vamp +pawan kalyan +co ded +grit ty +erup tion +se infeld +uni denti +ëĭ Ī +wor m +ac us +se ung +dun g +ro land +su d +di visions +ab lanc +shor test +j f +p oun +plant based +be to +tough er +mc o +don et +mark us +v fl +ðŁı ł +open ing +co ward +caber net +o xi +burle sque +sand ra +su mo +consi st +tho t +cay man +motor ola +gutier rez +d slr +y w +no bel +nov ice +moms demand +grun ge +sp or +d cc +pre sses +sli st +allot ment +voc ational +ft c +pu ja +lo ven +utt arak +tan dem +sh ep +come dians +anat om +cant wait +healthye ating +west side +mar gins +chi ang +asbe stos +stupi dity +proble matic +fit bit +: $ +ceil ings +shu a +protec tions +bio tic +beng ali +re sts +bien nale +tim o +cul min +e minent +affe ction +unbeliev ably +individu ally +canvas sing +wh itt +nov asco +chin son +h pe +go w +gloucester shire +pa o +thresh old +chev ron +s ine +we ther +pp ie +aqu ino +antwer p +âĸ ¬ +po on +inst af +equ ine +cinemato graphy +nbaf inals +vali ant +kil kenny +te rence +syste mic +sr l +p ound +made ira +pl ough +tre cht +mat ed +mp d +ransom ware +ph in +li qui +bb ce +boom er +i standwith +con ju +r te +nar a +foo lish +da shing +vier nes +br ite +da u +juni per +ai da +you now +ra zer +de i +repe ating +comfor ting +adjac ent +e to +ca sted +chat ur +mu er +syn th +san itary +mac le +independ ent +law ful +e erie +h or +ðŁĴ Ń +am rit +vel o +station ery +mu f +may may +contempl ating +elabor ate +gre gor +dri es +ac col +ภļ +schwarz enegger +ill nesses +day break +follow back +collu sion +electr onic +jo vi +hiro shima +ta w +hom ec +mic ah +qu itting +fro sting +ben fica +hel i +s ical +pic cad +corpor ate +ment orship +you are +sing er +shi va +ru ne +ing er +ri um +play able +doo p +wil low +ter re +ni p +at d +war bler +profession ally +er ase +proce ed +pedestri ans +mis chief +ben ding +alas kan +c kett +mo p +dd les +shut ter +ge ared +atene o +ma deline +g ations +o sha +der ick +sw ild +an gry +pat ents +hun k +decre ased +fr y +ðŁĴĸðŁĴĸ ðŁĴĸ +sal on +quant ities +d ario +ni gel +ku ma +jen n +happ ye +xx x +rex perience +pro s +au sch +rele ssly +ham burger +fuku shima +er ne +stat ec +ren d +may field +j one +lef ty +bern stein +sm il +gener ates +fore station +band its +ta yo +r ca +ac ci +rodri go +kn app +elo vers +vege tation +u ral +le ft +ħ ï¸ı +worl dre +sur i +embar k +w son +ba you +mu ller +mo vers +ðŁķ º +presby ter +l f +cre e +bat b +sal am +demonstr ations +an ec +n pc +it ics +to graphy +re inst +thur st +tal e +off ences +smart city +bro tha +ofthe year +in valuable +ear n +ðŁijı ðŁı½ +kre mlin +gra dy +town fc +guern sey +ma ha +contag ious +dre x +be en +( £ +nati vity +k tm +somer halder +comp ounds +íķ ĺ +" âĢ¦ +af g +ott news +h ound +fire fly +cil an +donet sk +volunte ered +ak ira +è ª +sing ul +st h +dro wned +mand o +he ir +ðŁİīðŁİ Ī +tax is +y uki +vel d +k ans +el k +ran ts +hash tag +t eng +ro g +a at +gru b +e ber +in india +colo ssus +sig ni +so ever +mile stones +der o +differen tial +phu ket +master mind +an gh +mel ani +bro ker +actor vijay +stun ned +continu ity +af fl +vo cal +perenni al +fianc é +in complete +hun ts +re issue +domin ates +tur meric +ro am +ri on +bag ged +nas sau +fu t +x ox +national trust +jo ye +san o +hearth stone +dis respect +le es +h se +siber ian +offe e +re stock +wolf gang +re gan +plan o +un wind +re par +mil le +] , +skul l +fat ally +concep tual +ðŁĮ ² +f é +ber to +b ms +u a +mag na +notre dame +le te +la undering +heartw arming +buffe tt +go at +pe abo +wind mill +v ac +continu ally +az alea +mem brane +can cels +make yourown +athe red +p to +tor pe +ðŁĺ ł +ðŁĴ § +sc ares +le aking +z et +pix els +ac i +kh il +marath i +ðŁĻı ðŁı½ +u la +tam u +chandi garh +z agre +aa b +pronoun ced +aubre y +sand er +pun ta +har low +ic elan +celebr atory +so t +unci ation +stru ly +mc dowell +deepi ka +remin ders +my stical +ct c +chat ted +s ica +bar gains +ch hat +ru bin +m net +oiland gas +pel ican +o at +mor ality +k our +i h +nu clear +gc u +ric her +vene zia +m ma +le ith +ac company +rich mond +sports net +ba ahu +smu ggling +mm i +ðŁĩ®ðŁĩ ª +twi sts +sahi b +.... . +amb itions +il lo +histor ical +fo rec +show biz +pon ies +chas ers +remo del +will ing +prince sses +am ple +cushi ons +ac les +lot r +da ch +an the +in corporate +new bury +ki ri +fried rich +ab v +ball ers +alber t +ðŁij Ń +let i +nan op +ci de +anal o +n sf +)) )) +griffi ths +valen ci +ro ano +fun run +babys itting +ca day +ent re +u ck +slu g +tic al +the sims +ro ar +car ney +g am +sto we +fi d +bun ny +sham rock +pe cu +mol ina +go cougs +con tributes +transform ation +mo y +v aj +sever y +antioxid ants +thir teen +sight seeing +l j +reversi ble +odd ly +hoo kah +nou vel +hal al +fe i +stab les +mul t +ho pped +bra ids +inter change +ghana ian +ww ww +eth no +con junction +ago v +ye ti +earth and +ts p +con serve +heir loom +metaph or +woo f +tor io +self less +n wa +em ilia +yl ene +y xe +gi ar +moder ating +pro bz +b fi +ne er +du mmy +hanuk kah +we bber +k v +eye brow +dag ger +su mp +ra ges +ork ney +tb o +hal sey +assign ments +tr onic +scri b +co on +an war +# âĢİ +jal ape +flori da +qu aid +haw keyes +âĻ¡ âĻ¡ +street car +ro g +dat lantic +gran ola +un changed +expect ation +Ù ĩ +mar lin +gu mmy +ðŁĻı ðŁı¾ +awareness month +oil painting +mu th +per ch +jun to +villa gers +mor g +che ated +web comic +the future +d ps +la kings +men tioning +vo or +ident ities +accor d +mc gu +l pga +rum our +massi vely +m pls +heal y +d ate +sp oli +re visited +on t +al and +scru tiny +lakel and +bl ending +< / +an kara +jami edor +metab olic +f ences +ann y +å ħ +semic on +oo tt +space ship +wack y +le ta +ap ac +she e +in herit +do res +ðŁĩ¨ðŁĩ ¦ +gent e +tw ick +ri ms +gal ve +de ville +king fisher +scorpi o +ow l +al ar +vari an +ðŁĹ ĵ +vene tian +star dust +then orth +q ing +har rington +consul ate +spectac le +ho bbs +tur ks +gre er +mat ing +ðŁİ Ģ +ðŁĮ Ģ +direc ts +í ĭ +pompe o +vo iced +la os +tz u +pro me +pri sm +mer c +fortun ately +bc fc +mcdon nell +not sorry +smi led +t ba +for war +mid term +dar by +we instein +up grading +wol ff +bron co +cab ello +ðŁ¥ ĩ +fi able +shar pe +bat tered +sat o +myth ical +instap ic +pre pped +eni um +e spo +di aper +explan ations +who pping +ragn ar +pe el +antibio tic +l acks +harri son +li sm +au l +qu ail +martin a +sent encing +sc ams +di di +tr onics +ãħł ãħł +go ff +za in +param ore +cha ined +clin ton +li ff +cott ages +em on +reve rend +consu mer +ce an +t any +lum pur +e bay +sto ol +ðŁĺ» ðŁĺ» +ta pro +h ath +modern art +just ine +prover b +app y +tra x +mani fest +am bu +nai k +pe pp +r sd +mer chants +kitch ener +shi fted +li zz +âĺħâĺħ âĺħâĺħ +âĢĶâĢĶâĢĶâĢĶ âĢĶâĢĶâĢĶâĢĶ +uto pia +tom o +ou ted +com ers +chiroprac tic +book club +cin dy +pro hibition +se uss +ë¯ ¼ +thin kin +rr rr +go fund +t ack +om b +catastro phic +ling u +guild ford +bo td +ॠĭ +plan ter +^ ^ +win k +kath mandu +sto ppers +smooth ies +re efs +hin d +bell amy +Ħ ë +waste water +vo or +nat l +! ] +re el +y ap +scoo by +work space +corin thians +bl un +obli gation +g bbo +dy son +cra vings +ell ington +dap l +wre xham +earthand clouds +uk runchat +positi oned +kal b +four square +jo ck +im pending +even ing +ath y +pro claimed +c ites +ann apolis +san i +mar th +ir l +accom mo +ka a +fin a +y aa +di sper +ec ar +bha k +will y +ðŁĺĢ ðŁĺĢ +mcder mott +mo j +gener ational +u said +train ing +lon ely +lo res +impe cc +âĢ IJ +beav ers +ma ki +he b +aap l +å ı +wolver hampton +leader board +me u +c fa +easter n +hu r +civil war +ou rage +hor ned +le high +awar ds +evi dent +gi gab +r ous +ma del +ro byn +ur gently +k ors +en as +heis man +bam bam +fab ian +f om +evalu ating +assemb ly +out sourcing +hun tsville +ðŁĶ ª +justi fied +cashi er +sp aper +buc keye +analy tical +illumin ati +au tho +o j +sha de +geel ong +wh ey +he aton +terri bly +ele k +un charted +sd live +moto cross +her mes +dar shan +dar lington +cash mere +gri pping +cilan tro +pun ish +... : +ðŁĴ Ħ +inst ance +der i +lo bal +muk her +sp ar +thin ker +fre mont +com piled +color ado +vig ne +sm d +whe ad +villa ge +le ek +formula e +ta res +persist ence +?? ???? +ped ago +he z +alzheim ers +vul ture +off ence +is great +suff ra +kick in +h mmmm +broad way +ï¸ı @ +art i +alli son +endor ses +ry u +lolli pop +soy bean +kend all +cer a +inv ade +( ðŁĵ·: +conver ter +car pets +ho bo +fr it +pe ac +es qu +ern an +ou f +an il +di ffer +ch ing +bre cht +sp g +daven port +stra va +sever n +n gos +stor ians +fe te +parame dic +j hb +al amo +sne aking +gold coast +roof s +isi l +depic ted +projec tions +nu mb +o ss +ep i +glu cose +zid ane +infin iti +íĺ Ħ +ran som +ton ics +fal k +g ler +ou tw +re ss +week ly +the on +n ole +ðŁĩªðŁĩ º +vol ley +sum mar +neg ativity +sam son +ye w +aus votes +ju l +ju dy +f art +pra yed +pal ate +multicul tural +double header +cycl ones +pier re +ãģ ¨ +âĺ łï¸ı +rt w +conver ting +wir ral +l ari +ir relevant +austin mahone +an che +ya an +sd f +$ . +explo ding +ulti mate +prof ici +gofund me +cell ence +ep stein +bul lied +sep tic +à® ¤ +lu mber +cu ff +vsco cam +pl or +ภ¥ +se ok +ro to +venezu elan +sor ta +spir ited +daniel padilla +team sisd +radio active +icelan dic +ðŁĴ ¤ +ver e +accommo date +shi pp +ot ter +ol ina +e go +su la +san antonio +de as +simil arities +âļ ¾ +y om +bro ward +å ° +can cun +veri fy +on te +candle light +ìł ķ +inf ants +az am +ðŁĺ ° +le ven +un stable +bloom ington +x ford +con tour +y p +innov ator +histor ies +po y +lolo lol +ex pires +cat alo +bill boards +an ab +el ic +novasco tia +fa ire +ìĿ ´ +rock well +gr ille +az tec +joh or +ur struly +fi ren +dun lop +id le +port man +jo es +tx hsfb +hol m +cham ele +under world +lo ss +ti em +therap ists +past ure +pa ste +ing now +vul can +ra gon +lar kin +o shi +ho co +child hood +umb rel +success or +kath y +iz en +° ï¸ı +share holders +ol ga +ai b +he ap +fl aming +ro u +air tel +rat t +z ane +vo w +thor ough +sn ag +par th +un conscious +ve y +new release +gh ee +croati an +facilit ating +swan son +astor ia +to logy +master y +ðŁ¤ ij +bil bao +trou pe +the ori +chey enne +ro tt +shore line +gra sso +master chef ++ ) +vi x +ellen show +as g +an ak +ku ya +safar ilive +debu ting +blu m +list ener +v ins +book shelf +smart cities +makeyourown lane +; ; +ðŁIJ ¯ +ri zz +on ward +bull dog +bear ish +vir uses +fri gh +lin den +we iser +sn t +gon a +dre sden +fl anders +cu k +wheel ing +ba u +atu esday +surf ers +swi ft +mc call +arbitr ation +aw d +mon c +b ine +at x +re fr +mi ro +po sey +n are +rit ter +âģ ¦ +play book +blow out +sports manship +s oooooo +malay alam +gri ms +bur bank +infin ity +sar gent +oit nb +joseph ine +ski pping +par kin +excur sion +semin ars +jo har +par tridge +post game +ll ll +blan che +temp ting +m na +lu ka +is ers +to ffee +bar ron +he mmings +sa e +go hawks +cu pid +li mbs +con se +un common +z ada +head shot +so ils +pione er +mam ma +sem itic +pan dey +jamiedor nan +spl its +vel a +son i +ra ff +t mobile +âŀ ĸ +pra wns +lit er +enjo yment +egg plant +tu b +cultur al +us ic +suspici on +sy cam +summ ed +ma du +ho ck +up wards +eye ing +ri ve +assas sins +âĤ ¬ +out fy +chi ves +t ner +la is +por ridge +sad dest +w cc +vick i +sna ils +biz italk +mill an +ðŁĮ į +sam oa +j ing +mi key +gu j +chel ms +eli gibility +arma da +thro p +surger ies +ãĤ ¿ +mo hawk +ex its +me m +is lington +c me +land fill +kait lyn +ðŁİ ¼ +combin ations +tomorrow land +ver b +cor a +pre cisely +na om +ðŁĨ ķ +shr ink +sof tly +merce de +mand el +poo dle +ball erina +sop h +jux ta +y at +ary an +hesit ate +lo wered +gu lar +dungeon sand +ron an +my ri +sp f +men opau +gra sp +pa thi +fe asi +fla w +shi story +ste ward +gg le +fay re +cli que +credi bility +yo g +sec tion +mu sko +se ville +no tt +cal m +mate o +indic ted +fi ba +by l +lin o +u kin +!! # +enig ma +siri us +bu sc +ðŁį Ĭ +mac kerel +psal ms +a at +tomorrow spaper +ðŁĺ ĸ +p fc +........ ... +shre k +mul let +o sh +danger ously +immen sely +am ur +ðŁį Ĥ +pro por +sy a +london marathon +abo ve +obli gatory +pro v +ra cha +alex is +pri mary +sh h +ether net +d stv +cou gar +un lucky +ni l +steak house +mel a +fc bayern +cause way +ca therine +fluore scent +nx t +to kyo +au sp +releg ation +qui zz +shored itch +proud tobe +promo s +inter acting +home brew +da esh +w pg +stead ily +provin ces +bal lots +i ah +al to +< << +you u +ri ley +prefe rence +tra verse +incen se +am munition +ho dges +# @ +hail state +tart an +witch craft +vent ilation +liber tarian +! âĢ¦ +ow es +% ! +ong chang +bru shing +le ic +fi ber +under attack +down load +ex pir +hy o +pompe y +mc bride +y ag +stre e +com bat +ten ding +ai ra +gug gen +ab ra +in na +fli ps +aw al +m ach +dol lar +inspir ations +z um +o du +it ty +video game +aqu aman +har u +bel fast +je b +but ch +us gs +calcu lus +go yal +mor gen +x finity +stand up +contrac ep +sab re +na be +in secure +gener ously +epit ome +l w +t ca +narr atives +don nell +pand as +ber gh +tu t +ker al +fel icity +br ampton +quinte t +nom ore +ðŁĶ ij +lo i +alham dulil +ðŁĶ¥ ðŁĶĹ +ston er +shaw l +clin ical +bren dan +gon e +fla wed +tri ppy +j g +al location +po aching +ve vo +mo cks +lef tist +bon uses +condem ned +abil ity +st ating +microbi ome +bio logist +for you +wahl berg +ss or +ift ar +w ul +ÑĦ оÑĤ +pom er +me me +ver te +tre ll +tra it +in let +hormon es +deliber ately +vill ar +battle ship +p bl +tw enti +ho kies +dal ail +say a +may fair +han s +die ts +⾨ ⾨ +od in +hot spur +pap i +k ana +k amp +fin na +flo tus +ti ans +unic orns +tribe ca +chang ers +fore ground +out a +inv aders +gett ys +tomorrowspaper stoday +mac millan +hand written +w fp +u de +state of +base d +âĺģ ï¸ı +cas m +psy ched +histor ians +fol d +d da +ag grav +p ans +green way +au sv +ðŁĺ ¶ +shradd ha +inde x +be sti +zim mer +t ness +eye shadow +ot te +go ts +distribu ting +pro min +yo l +ace a +tram rahim +hoo per +supre me +jam min +intu itive +quali fications +sli m +sid di +jay ne +tri pping +g tx +pun s +e manuel +om g +mid summer +in to +succul ent +ri en +new mexico +o or +hoo king +in f +ðŁ¤ Ŀ +flir ting +na hi +g friend +t ps +hel ix +z s +on ie +ct f +kri s +irresi stible +fla p +ðŁijıðŁı» ðŁijıðŁı» +us wnt +ru d +ram ps +pin oy +ot w +lol z +low ering +favor ite +t mc +phra ses +her mi +aver aging +em br +ben o +estu ary +sle eve +ribb ons +ta sh +ภ¹ +x f +aw gs +sun ited +brew eries +anir ud +pun ches +ol die +ip ads +wi fey +land lords +d ji +gun ner +íķ ´ +tex an +ex op +cas sandra +s off +ðŁļ « +igh ton +bak ers +awareness week +v all +ear p +bts bbmas +apologi zes +âļĵ ï¸ı +was ps +states man +snat ch +watch dog +ra fi +after party +spi ke +j er +peri ph +r nc +mu ll +le en +shi es +li eu +urstruly mahesh +mer ton +de sai +shi f +ðŁĮ ± +pe dic +gos ling +arrang ing +ww g +gen y +you uu +netfli x +e ttes +k wi +bernar dino +am iga +Ø ¨ +kashmir i +t ings +emer itus +de cat +ab domin +dc i +pha ses +d jan +be am +op ry +i shed +the ellenshow +the st +habit ats +to ons +mclau ghlin +ri pper +micro biology +tal aga +clu eless +ss u +cro che +bro mance +longe vity +zagre b +prev ented +tra ve +spo ilt +darry l +migra ine +al cat +dd dd +vi v +ser pent +mat tel +jam a +con quest +î Ħ +sam sung +presbyter ian +ket ch +fire fox +mo tif +le c +cho pping +cher no +j ann +ðŁIJ ° +pro lon +wake up +conver gence +mersey side +heart broken +lo oming +hal lucin +mai ze +commun ism +mo h +twitter storians +serge y +res eller +favor able +ed gy +re iter +mal aga +live me +ka hn +pul sion +big g +kim kardashian +ati o +tyr anny +ru ption +q ant +pro ven +by z +pu shaw +kri stin +e er +tar dis +ri z +awak en +mi ko +un documented +path finder +indirec t +resemb les +h ler +conce aled +scand al +re im +d nb +cr itters +attend ant +apprentice ships +aa u +scre amed +l su +fa h +har bour +ed d +bat sman +li ss +mi sha +spani el +it f +advan cement +fa c +close up +cecil ia +medi c +narcis si +lav ish +gi ac +ma ys +le it +wine wednesday +pushaw ard +let to +curren ts +bug atti +out ine +w j +un do +ler osis +devo tional +ðŁij « +on na +fais al +sa una +himach al +am ii +à® ® +di zzy +screen writing +ph x +sp n +ick i +ag irl +fi shes +wb z +pi m +bo ar +ac id +! .. +rocke feller +n ga +dra stically +simpli fy +dru mming +autum nal +gur mee +lor de +jo ann +give up +b our +am ura +der land +sim pler +wat son +tri dent +concor dia +bel lum +bre k +dum plings +vi on +dungeonsand dragons +sp ri +ascen sion +wil datlantic +u st +rob ins +legi on +insi st +jar o +gue ss +so b +bigh it +pool side +negoti ating +mc gill +bil d +techn icians +miti gation +ajay devgn +b to +ant en +cosmo politan +ðŁĺĬðŁĺĬ ðŁĺĬðŁĺĬ +patri oti +temp er +promen ade +nav ajo +nam m +wrink les +dc fc +le ach +bru nette +r f +cout inho +al ti +tradition ally +op tome +na z +accord ingly +rec ard +de ets +sw ell +po sure +whit ening +strang er +illi on +here ford +u wu +ro bber +cotsw olds +cl en +gor ge +nam aste +re lish +gri ff +adren aline +bla sio +val e +ê ² +toler ate +rail minindia +jen sen +ho ven +el lu +ob sole +eisen hower +unidenti fied +than niversary +body guard +Ø ¯ +i dge +sch al +stock port +sn i +re taining +po po +pix ie +oli thic +ki er +ha jj +sa z +cor bin +!!!! !!!!!! +v it +me gat +de h +circu it +af fleck +theore tical +hope less +u ab +slu mp +b ice +jam med +let stalk +can i +side ways +labyrin th +re fs +ha hn +jare d +ðŁį ¹ +jam bo +ph yl +enhan cement +c tr +ful lest +se ye +do ba +cho ic +yo s +cb j +andr é +re watch +pri ma +doctr ine +for gets +u hm +ar ound +u le +art lovers +shi raz +har th +ex tor +Å ¡ +unexpec tedly +eli us +y x +em my +se ac +ðŁijĩðŁijĩ ðŁijĩ +correc ted +com bu +wom anc +cou gh +what son +publi shes +divers ity +back bone +lock down +mesmeri zing +nor te +ma b +desig ner +í ģ +ra gh +mole cules +get outside +the beatles +semicon duc +nach o +lun es +ham mers +sul tan +o on +fe ren +att ach +ar qu +uttarak hand +s ash +; - +tre ad +i ko +ar thur +scandin avian +r ation +ga el +charge able +fish y +v ma +hand bags +char a +ay ne +de fam +sett lers +qad ri +pal ais +in wx +apocaly ptic +poo ja +a es +at ories +proof ing +n lp +ts la +v ina +li do +dee phouse +informat ics +v v +pp ings +di ss +à ¯ +uhur u +st ony +betra yed +b aff +my ra +as pen +allow ance +tam ara +ci f +cor bett +ser ge +di go +ambi gu +pain ters +p cr +p ca +nom s +lo ft +ve e +opend ata +ðŁIJ ± +alex andre +identi fies +fantasy football +re production +brom ley +ware agle +mm er +p ss +cu es +ay at +hut chinson +sar ac +jack man +ira h +ap ink +col s +aussi es +ex ecs +day ton +ðŁĻ Ĩ +im v +har am +chuck le +authent icity +ar do +incub ator +ภª +photo shopped +embrac ed +fight for +gor man +zz zz +schol astic +cri sps +te apo +mid night +ga ine +col lier +s ate +de tte +å Ń +imag ine +i ff +tw ili +i fication +teat ro +nor ma +es ur +emergen cies +rise up +r inger +hass le +cait lyn +tranqu il +vers a +se b +over look +gin i +bo go +se re +may ne +henri k +contamin ated +rhapso dy +pro portion +wildatlantic way +âģ© . +organis ers +tran e +stand ard +sper m +laun cher +ric ci +her ts +paper work +showcas ed +mer yl +pen a +p imp +disa strous +^. ^ +phar a +x is +fron tal +sw irl +sp ills +swag ger +smart watch +sizz ling +savi our +cat ar +bb cr +refurbi shment +dr is +citro en +absor b +patrioti sm +il leg +chro mo +fresh ers +ru s +lim iting +ef ish +down ed +man dir +hazel nut +p all +mac on +disappear ing +quali fies +bo on +bar racks +am ine +gen dere +ðŁļ ĺ +j es +ãĥ Ń +qu ito +middle weight +sch au +quad ru +aci ones +limit less +ðŁijĮ ðŁı½ +ch man +ar av +regulat ors +it up +batter sea +mil ford +g z +tic king +gh ou +cru shes +tu tu +dread ful +fam ine +for change +dalail ama +ðŁĴ į +whit aker +hash mi +h us +vo d +bet te +aa ah +iso o +ðŁ¥ Ī +ha ar +la ine +b v +all day +spr out +indie games +free bie +gree ks +but ler +ill in +ha al +ware ness +si ma +public health +gam a +wa a +oun g +goo oo +okin awa +off enders +im pose +ho c +young ster +story teller +sc ap +figh ter ++ , +whit es +music monday +re za +go ducks +bri a +mi um +cas per +cru mbs +a ad +marti alarts +ch p +ri gged +tn g +harve sted +sa k +do jo +mill wall +b nw +oc d +histor yof +t mr +si rens +fan ci +caregi vers +vir a +son i +recur ring +acknowle dged +ðŁı Ł +oph ile +bu cky +stre ssing +roo k +di gger +vi val +san do +fle et +si ers +sel caday +refre shed +anti fa +a que +po lo +disappear ance +de mb +âĮļ ï¸ı +ren ted +ber ger +g mb +cu la +ss al +goo dy +u hh +marcel o +w anna +soft ware +shop small +turt le +tom as +fri sco +ðŁĺį ðŁĴķ +jim enez +c su +day z +an do +wyn ne +choreo grapher +cerv ical +trail blazers +ed g +zend aya +travel blog +el s +whole some +co g +lab out +ar ney +del le +su isse +ma si +ine se +om be +fi ddle +re claim +pa u +wat cher +sla in +ber ty +opti mum +el ites +min is +tur key +patro ls +ger ard +au reli +wild ly +wal tz +br gy +w ob +cre st ++ ++ +ve z +fro sted +davi do +the x +param edics +p into +han k +du pont +ur g +fo stering +micro poetry +spec tre +---- > +ne uro +fri da +music al +galve ston +e ffic +sc ape +pal azzo +th all +pro visional +p js +au re +ðŁĶ ľ +mam amoo +kit ties +cre e +wa k +lo ool +lu pus +cn blue +à º +ðŁİ ¬ +rac ed +tro se +om as +stri de +co ors +⤠µï¸ı +in comparable +cy ril +broad er +arec lipse +ðŁį Ķ +inter val +ti ru +co working +w aco +a ham +a bee +flouri sh +the times +ol ini +kick boxing +lu cer +at la +as un +casser ole +mi aw +lobb ying +jan ice +cir que +re flex +le ary +sanat omy +tem pest +se mb +mur dering +us av +ro bo +on et +p cc +nati ves +life of +sa ha +ruth less +rel ates +appeti zer +pye ongchang +nor d +er u +a thing +ug ly +pl ying +bran ce +organ ise +kend ra +dat o +chees es +par ma +burn out +a stra +pre toria +adjust ment +uk u +sl o +li ken +fav ors +cli ve +be ets +snow donia +go tv +sy n +open house +pan i +portra yed +sl ated +me cca +ren al +supportsmall streamers +staf fs +da o +bi ker +vik tor +tit us +admi red +ðŁĵ ± +hurric an +he ats +gl ory +photo genic +mer i +de por +burn ham +or angu +dj ing +impre ssionism +ign ition +ca i +w ynn +de pe +cove ted +colla gen +sau s +or nam +administr ators +ss on +nh politics +hahahaha hahahaha +aspir ations +r gb +swol len +so we +sc r +diver gent +hou ghton +han oi +d ory +ni ki +land ry +b cci +ðŁijĮ ðŁijĮ +is mail +tri pod +her d +bhat t +dress age +tab by +ingu ish +hur on +à³ į +à ł +to das +evangel ical +chor ds +st john +slo ppy +marty r +face book +ali ght +sen sei +kath niel +r ites +zi one +u o +revel ations +weight lifting +pan o +nc wx +ac ton +à® ķ +Ø ² +som a +à¸ Ĺ +respec ting +mar che +fore man +be tty +ki k +shi bu +po on +argy le +k swx +et z +mar bella +brac kets +stand by +fire side +defi ance +v ex +britanni a +in habit +appo int +piyu sh +le ash +sci ento +fla sk +sen na +> : +at roc +sand erson +id lib +dhan ush +ðŁĺ Ļ +en thr +hit ch +de dly +al ley +dor k +mon do +cudd ly +mis sin +ye sss +night ing +j pn +w ary +ump ire +ma z +ê ³ +bab s +ĭ ãģ +stan ford +posse ssed +exce eded +ðŁĶ ¶ +wall art +tra p +j il +hi bis +sp ying +scri be +khali l +trans lator +lu mb +di zed +ch c +super vision +shut ter +ja g +_ * +yester days +ms f +hi hi +gonz aga +gille spie +vive k +ec static +this morning +ch us +ed es +ston ed +be es +ðŁĩ¹ ðŁĩ +tur in +ho ver +at rics +ster n +sam heughan +auti sm +mi ya +eye witness +writ ings +travel tips +chut ney +px rtg +keny ans +my stic +k rit +/ $ +red head +world ly +am us +op la +le ve +gab bana +se en +o clock +gang a +keen an +sc ent +ol dies +go green +corner stone +comp ly +con cours +ðŁİ¶ ðŁİ¶ +ha an +con fis +aw son +cle op +î Ģ +su zu +sau té +al gar +subscri ber +este emed +ãĤ¤ ãĥ +worth while +mel rose +flo ck +bri ghtly +viol inist +p ere +sli pping +and co +si gh +ha van +cu lo +m sa +fibro sis +matil da +ra fting +aw ard +ë ª +mm mm +ge aux +ste iner +sin n +help ers +beet les +ai mee +tai wan +pistachi o +mac beth +m zan +descend ants +on sale +in r +il m +grou se +sa ig +mo w +bi gre +adjust ments +tu la +mathe w +transl ates +mu h +bol lah +ðŁĴĽ ðŁĴĻ +amo res +ab outs +bomb shell +bla ster +x avi +s ns +k roger +ga ther +erad ic +daf t +chem o +ben ches +ðŁĩ© ðŁĩ +ut v +our a +n ko +gator ade +biaf ra +ok state +im danielpadilla +dom ains +open ingday +kid do +do i +ric e +day care +mac millan +ba thurst +cheer leading +ðŁ¦ ģ +cash back +k won +hob bies +exem pl +ries ling +âļ ª +ag les +ny s +every thing +nav is +ad di +magne sium +faceli ft +ark ham +grand es +extre mist +don at +vit ality +pump kin +be tta +sl td +arti san +li by +pe aked +ah hhhh +mary am +assi m +un sc +ment e +al aya +low ers +ar as +gri ev +le ip +gr ati +cri ses +spr ints +exe cute +w to +ms d +mag ical +re viewer +spark les +juke box +ðŁĺĤ âĿ¤ï¸ı +pay back +licen ses +dun kin +bel t +lake wood +h ateful +bud gets +rev amped +ph erson +ky iv +went worth +ro sen +cru ise +gi ggle +def star +assassin scre +ym outh +win kle +w fc +band wagon +b kk +w iring +kear ney +south side +pe tit +! ðŁĺį +nor dic +mir za +mu gabe +v l +scon es +k tv +sand al +du c +m alls +ðŁĴŀ ðŁĴŀ +it c +al ay +im pair +un rest +flo ss +c é +ab ou +var ying +muse o +ser ver +di ya +hibis cus +ero y +mer ritt +fin dom +f pp +un usually +go tt +conting ent +ali aa +ball on +jo l +hi ked +zy me +ay r +ag n +ga z +perio dic +spar ty +practi sing +lin ton +tal is +cy pri +womanin biz +radio disney +ðŁĮ ¼ +jump ers +endo cr +ðŁļ¨ ðŁļ¨ +and on +shar apo +mi er +ma sonic +fac tories +vi en +bb ers +ìĽ IJ +hol d +ke bab +be ak +approach ed +ac milan +mun ro +ko sher +excell ency +negoti ation +walt disneyworld +cr ouch +te asing +suppre ssion +en ya +b ce +transformation tuesday +cal lie +vis was +p gat +ic ted +end ings +esc u +recru ited +it fc +collabor ations +g ino +snu ck +ausch witz +i fc +x ii +ke sha +ger vais +clo ak +x l +sa ad +prob ation +pre cau +mac in +anasta si +le k +e azy +daysof code +mariah carey +yo g +stit ched +boy friends +sh ar +ph ile +ag u +twin kle +phi shing +week ender +ic ton +gurmee tramrahim +al ton +l eness +all an +pen ultimate +kry stal +go u +lan de +dis mant +ab using +nor se +pat erson +ed mun +ap an +xi umin +sk el +cat walk +re act +wal led +t angle +br yn +ve to +super moon +cas ablanc +appreci ates +ski d +bo th +catal ina +ele ague +cyber monday +cau tious +ðŁ¤ ĵ +nov o +hamp ton +ha ye +jose f +var an +lo bos +roano ke +orph ans +tt in +squ ads +ishqba aaz +black panther +e tu +k sh +cru mble +cess na +reli eved +scul ly +pollin ators +explore canada +ki es +kam loops +kir an +pri mal +sett lements +hot spot +brain storming +ce dric +bi ennial +sh ant +âĻ¡âĻ¡ âĻ¡ +do on +hear n +walk way +fe m +ve al +deport ation +tox ins +elimin ating +descen ding +by the +bla sphe +ha sta +comple ment +as cent +ri ga +provo st +âĸ ª +wee ping +anti semitism +employe e +unearth ed +pin o +natali e +bla d +ang ola +lock heed +in ian +ag r +ni ster +im pala +m ke +fan atic +âĺħ âĺħ +ðŁij ¸ +lu ch +simpli fied +gall ery +econom ic +cy borg +con i +sel ma +in ception +ko ala +dv ds +cre sted +m mor +visi ble +n sd +ðŁĻĮ ðŁı½ +w under +refriger ator +re opening +e era +carou sel +as p +balli stic +victor y +mo tive +tre y +sharapo va +si i +mon ter +int end +west chester +sp e +cy mb +vi dal +ll ama +uni v +fin er +crafts manship +jazz fest +b ch +ag gio +n cc +lamb da +tranqu ility +cis co +ba den +so bbing +of i +go ta +ru mored +war med +ore an +ac ton +mar ci +gh ani +âľ ĵ +as sorted +pembro ke +pen elope +da f +at ty +aim o +pretz el +carni val +than os +ko chi +mer sal +ham radio +ar twit +cas c +guer rilla +kush ner +k app +al ise +todd lers +steward ship +o tti +ter ri +tem pe +rest less +vit o +zay ed +rsp b +pi on +hi ppo +haw thorne +in as +am ily +nut cracker +lo p +d ali +tro pic +ðŁ¤ ł +ul o +jare dle +py rene +pale o +usa ir +m ould +it ated +gene tically +biom ass +ðŁĩ³ðŁĩ ± +do dd +practic ed +monarch s +un manned +m buhari +am al +photo gra +ko ol +bren don +ju ices +cu re +world bank +poin ters +ðŁĴ Ŀ +tur f +le ds +bor ussia +bapti sm +warwick shire +moun ts +gay o +be gg +co pied +asi ans +k g +moder nist +gi d +front man +concentr ated +y t +sc avenger +iron ically +adi c +ps n +ðŁ¥ ī +cultur ally +yu v +mac arthur +fertili zer +be withyou +ri gor +min ors +z oning +âĸ ł +ri r +adole scent +vin ny +ren g +sand stone +gu et +we sth +ple dged +lac ed +sp ide +v ai +ty coon +seiz ure +du p +appalach ian +ro k +cathol ics +sey chel +posse ss +la ger +jo di +cham p +stra s +d ina +cent uri +cal der +blur ay +ðŁĩ¨ðŁĩ ³ +mo do +an nette +youtu bers +chap s +ang ling +label ing +a qui +pk wy +ly le +bi sexual +lit ur +dug out +li bby +grey sanatomy +sub stances +august us +rall ying +fi del +ing ue +äº º +hallmark channel +tooth brush +m á +adi rond +ag gi +ðŁĵį : +cru sade +tax ation +k z +i ver +dou bling +room ie +wa b +en rolled +az on +a ju +grand children +as df +ðŁ¥ º +mat ic +ough ton +utili ze +ðŁĴ £ +pon der +rais in +dys function +co bain +butter nut +e man +su red +dri an +and friends +with the +on omy +heine ken +bri dal +leader ship +pyram ids +deutsch land +jo cel +bo wel +y qr +horse power +be acon +ing eni +gra dient +fer mented +mo om +thing y +pot assi +wrist band +bor d +bo died +ðŁĺŃ ðŁĺį +ma pp +ka u +cyber punk +ph ish +loo king +co ates +ap ur +am ie +uk labour +at in +g la +adop table +shel by +v illi +ri ya +m ingly +cli mber +bumble bee +ðŁĺ ¸ +c sd +âĿ ¥ +hospit alized +c ki +hat er +ch r +re tina +it a +fan base +beat rice +gwy ne +go ss +fo s +favor ited +swachhb harat +mal ade +mon mouth +" [ +si van +sh hh +command ing +sains burys +wee d +g man +ss w +rep tile +iv y +tro pics +roll ers +over cast +ex position +masquer ade +man crush +wa ist +spr inter +sle et +le vin +j pg +_ ( +o pel +explo it +ap a +po we +wrec king +jong in +or b +er ick +bo sco +pra ising +ber tr +to wing +in security +ku t +resto cked +rr p +prescri bed +trafal gar +per t +g ases +app rais +g har +music als +âĸ¬ âĸ¬ +mc fad +ag ony +conditi on +equi p +shi k +atra vel +ðŁĩ¿ ðŁĩ¦ +ke h +abduc tion +pe oria +wil kins +g ms +as d +ev i +ðŁĴĹ ðŁĴĹðŁĴĹ +u z +mo c +halle lujah +guad alu +lou vre +dra wing +go ve +ph ant +fri e +web dev +program mer +z able +games com +clari fy +li th +kin ky +âĿ £ +labour doorstep +son ata +ju ris +mai den +vi adu +buch arest +conditi oned +capit alist +u de +ps b +sp ca +lul la +footh ills +kay o +bon d +wom b +roun der +ce sar +bur sts +ap ra +sw oon +sab rin +fra grant +cle arer +ku brick +cli max +jour no +ag le +ðŁı½ âĢįâĻĢï¸ı +poo ch +hal e +sol it +sal mon +organis ms +bron son +art en +hodg son +alo ve +vent ure +bb i +ae a +ðŁIJ ¢ +ld n +d nr +o zone +el las +man ny +azz ur +un beat +tru ffles +th ong +ma ñ +las ers +ley e +gettys burg +back packs +or is +ma ison +craw ling +la bra +cl ing +dra gging +ste al +dou bt +de van +ck ers +agent sof +photo bomb +elon musk +abo y +dist ances +story line +sp i +nor than +europe ans +wh ale +ser pent +ðŁļ ² +fi or +tr it +ox o +awar ding +class mate +su fc +smar test +rich es +pr k +big foot +ar mb +bi polar +dw elling +om ars +k wan +gri me +m eng +freder ick +navar ro +sorry notsorry +jaredle to +pa ve +sl ack +barn sley +att ar +evic tion +accumul ation +o ir +cat chy +wel ter +vik as +has see +nik ita +mo yes +mathe ws +shi v +gat wick +pro filing +compan ions +mar rake +an tics +ðŁĻĮðŁĻĮ ðŁĻĮ +se se +bo i +bart lett +poison ous +ab uses +ym m +kam pala +guggen heim +imv kohli +dol om +bre e +thro ttle +gare th +fitz patrick +un ya +par ad +mar got +j nr +we a +potassi um +p nc +disgu ised +cra sh +ren ergy +ill ic +coup led +ni els +ci ones +æĹ ¥ +im ent +despic able +d ye +what cha +conne ctions +paralym pics +gaunt let +wait rose +suici dal +star ship +vap or +st ou +law maker +coo led +si mo +then o +offro ad +ja den +bas que +vick y +lu kaku +centr o +tri sh +strate gist +medic ations +hor st +b fc +gra il +sharp ly +ad itya +tom b +kau fman +tri pad +sam ba +pastor al +brit ney +sag an +hill side +mas ons +sar a +z one +x u +to tes +rob bie +app en +mon tag +der o +short film +charis matic +tat ors +ki ba +and ri +al arming +split ting +ic ar +th ug +scari est +sylve ster +an an +u trecht +a difference +me ade +bu ster +air strikes +cu ffs +account ants +ðŁĺ¡ ðŁĺ¡ +new t +bo tt +issu ing +cl ancy +wwen etwork +kyu hyun +rese mble +pajam as +sin k +kin ney +sul ph +or k +li es +la gh +or ton +ra hul +d sc +we will +re am +collo qui +shar ia +hec tic +sar casm +land er +tm z +endor f +ro z +ham mered +fri s +w adi +pope francis +he it +flash light +un born +op es +hol iness +ðŁIJ ¦ +nach t +im sa +gr acing +bj p +ver ts +c sc +home owner +a que +bigo try +anni e +bag h +âĿ¤ï¸ı ðŁĺį +car i +thom p +dispo sable +cardio logy +pat ented +hh hhhh +ld r +stephen son +cro res +fan ning +cli mat +ðŁijį ðŁijįðŁijį +ðŁijį ðŁı¼ +aer on +piccad illy +bank rupt +sil via +emplo y +don ny +commen ting +screen writer +io ta +ce an +anc ers +tu an +street wear +ठ¯ +sk ine +esp a +asi f +os ce +she ppard +more cam +bott le +der s +orac le +google play +aver aged +edmon ton +steph an +sister hood +cru sted +stag gering +methodo logy +congress woman +c abo +tri ggers +mil ky +gli de +tooth paste +room mates +nu ff +gu am +sprink les +alternati ve +wat fordfc +uof t +hal ey +cont acted +bun dy +pro stitu +gh ar +pre ston +on site +hil ar +g ts +c att +hamp stead +? ?! +ðŁĩ§ ðŁĩ +bbc qt +aless andro +resi st +ma idan +t ko +shad ing +pin up +gal lo +sin u +at ec +fun k +ac lu +stri des +rhy me +wet land +bbc springwatch +t ins +wild card +st our +flamen co +pau la +onto logy +gang sta +am ade +ãĤ « +t bs +skelet al +run ner +jard in +harri er +hun ted +z hen +believein film +de mean +au diti +re start +chon dri +âĿ¤ï¸ı ðŁĴĻ +mcla ren +ga b +sh um +au sa +lewi sham +y pg +k jv +fur nished +dor o +bon ded +mor ty +lat itude +_ ) +lo va +water ways +vin ai +shor th +drun k +c ay +ay ana +kap lan +capp uccino +spr o +life boat +has bro +spol ice +tor on +do ing +dam n +sh ree +foun tains +ent ation +mar u +boar der +to pless +j ada +chan ning +ul ls +en closure +gib son +fractu red +brit ton +à ¶ +t ous +por th +dra f +tra iling +mar gate +eli fe +down ward +lin n +gla des +girl power +ak rish +u ki +ron da +ts c +appreci ationday +vis ing +lo om +ðŁį ³ +mex ican +ar gos +y ya +jad ine +south port +d end +si sta +rede em +men g +bra xton +antioxid ant +s key +mp g +fin ding +vibr ation +ce u +kh art +di mini +cl ine +shel ly +hin es +ī ï¸ı +to pical +no ver +ma xx +prim itive +illustr ate +b ounds +tren ton +join tly +breed ers +u chi +wakeup america +b ada +ðŁĹ £ï¸ı +gu acam +sp heres +pere gr +youth ful +lo lo +bir min +t ly +jeremy corbyn +defe cts +co sm +a rent +v aa +bag els +medi ac +cori ander +ic ago +g haz +ab bas +re model +struc turing +pu m +out law +ad ani +r bc +gul ls +n li +confu se +ðŁijĩ ðŁı¼ +vil a +mcnam ara +correc tions +mug hal +ser i +re gain +ss b +lea ve +haha hah +gran de +di stressed +re chargeable +ho a +hou sed +sti l +attribu ted +opath ic +di ps +pri t +head phone +conclu de +pil o +he t +ut sa +nit in +je m +sni ppet +tutor ing +op er +sun k +en sla +cha u +ac orn +quinte ss +ran kin +affili ated +our lives +cl int +se ater +isa ac +ba shing +sme ar +nur se +doo dling +" ; +sa ku +atroc ities +im am +g fs +viol ating +comm end +brad shaw +er ville +b illed +b be +thul hu +i phones +moo se +di os +re w +me thane +strang ely +whis ky +ti ghtly +spiel berg +radi us +notic ing +wi f +ig nati +i fa +ap is +w ali +ha itian +bu shes +y z +v l +ex ited +asse l +tru ec +dom en +ash er +in king +newyear seve +hend ricks +bat i +ìĿ´ ì +rich ter +mon santo +con line +agre at +ðŁ¤ ¯ +master pieces +ar n +rough s +cle ve +se v +fashi ons +to ya +sh ail +cop eland +aqu ari +dec als +are you +y aya +a str +fon t +ml m +ar ca +pp or +pol lock +xper ia +conserv ation +chain saw +ag gie +?! ?!? +si le +sh on +ìĹ IJ +note books +marque tte +de us +bb led +spic er +mc cabe +nor wich +modi fication +boo sted +stru m +sales man +bang le +nis san +hez bollah +brea sts +a af +anth us +sk er +ow ed +her os +gi fs +fo sters +eat ers +du es +_ / +lymph oma +sf am +me gal +afri di +ag ic +p amp +jeal ousy +ðŁijĮ ðŁı¼ +calcul ate +napp ing +g ale +ðŁ¦ Ħ +lub bock +assu med +ren ting +íĥ ľ +subur b +ãĤ · +tech nic +u cla +in front +gar net +ster oids +stri ving +ho war +mo ver +le ton +bull do +is in +ci ao +sn z +fore front +d ams +mid wife +ma wards +cla pton +we in +subsi dies +spr oud +rother ham +phan tom +ar ach +spi el +rac ket +sel amat +no on +l bc +enti ally +ðŁĴ ¸ +sil ve +m oud +kine tic +y asi +ðŁİ © +o ol +mi ku +i za +fer a +flo ren +barber shop +groo t +z est +ne ars +stan is +z and +police man +juris dic +form ations +appar atus +sp d +arti fact +to sc +motiv ating +womanc rush +re dro +diagno stics +ra za +out fitters +el xn +dod gy +ry n +sh d +ortho don +ol de +jay anti +bal ances +quic kest +can ton +friday reads +! * +na a +a ak +ðŁĶ · +behavi ors +rasp berries +ä » +polit ical +cam il +å ľ +di k +ast ounding +lie be +novel ty +tur moil +sul ly +spring break +hon ouring +cc g +ðŁı Ĵ +my little +ky c +pro ms +ðŁķ Ĭ +à ¨ +bi ge +av ril +ðŁĩµðŁĩ ° +mari on +as ants +sur ya +oc tag +luf than +ac ron +fayette ville +ti que +love s +en ca +de kalb +ta ver +de vote +aux iliary +joh annes +tread mill +ay an +qu r +donald son +cher yl +" .... +s ven +kir sty +gun ners +ra dish +o ahu +v sky +i ble +con course +b ps +elo qu +ash ford +te bow +roblo x +ma da +dri ving +th day +spro ject +m ms +band ed +. !! +libr arians +flan nel +intoler ance +her al +ç µ +neme sis +list a +tar ak +cry pt +star plus +vish nu +sc ale +cr is +% ), +j illian +regg ae +pegas us +ol in +ip ment +man ic +l fc +godd ard +ite am +parl our +anch ors +lee minho +talla hassee +ant it +d ho +kid ney +y ash +batt led +az ad +gar is +faul kner +sni ff +papar azzi +ed m +phy llis +con tested +aa ay +se ca +k ton +vel ve +rain ier +for um +tam pab +ho sp +trac tors +ox fordshire +no tion +guang zhou +ðŁĺ ¯ +ref ill +wednesday motivation +sli der +mukher jee +pr att +fon taine +alph on +af ar +ts i +pest icides +fi ends +mo cking +bra w +tran sat +do ses +co res +hom ophobia +docu menting +zlat an +con doms +s é +sun set +kun st +ton ga +ภª +v ation +sp ray +chow der +ra ps +palla dium +nor wood +music history +hoo ker +si si +osp rey +ph ys +conce ded +bob cat +ar mad +ze it +Ù Ħ +ðŁĺģ ðŁĺģ +mer idi +ðŁĩ· ðŁĩº +corn wall +! ), +touch downs +ze it +chal et +mm m +al che +gor illa +fo ss +ati ku +lumin ous +ivan ka +be ek +sta res +sw iss +âĿ¤âĿ¤ âĿ¤âĿ¤ +scru bs +me ath +gusta v +jo gging +confe tti +as os +ers fc +breit bart +applic able +autho red +ya ho +h in +displac ement +j v +ðŁĮ¹ ðŁĮ¹ +ot c +non profits +diec ast +gu sto +inte stin +c ages +me en +lu kas +moon ey +ðŁĺ · +very day +tor ah +is sion +wa c +lever aging +ish able +cu se +le wood +may an +turn table +ju ice +tru sty +tu p +eti quette +supervis ors +stu n +gu zman +confe ren +ric o +fe ast +back ward +pol aris +mic he +jo g +h ing +field house +vel ing +sho cker +esc ence +ठ¾ +vi be +anasta sia +mar ched +kill ing +Ķ ë +fe tt +exop lan +... ( +snow day +lo h +ir ani +la khs +del a +po caly +boom ers +dictat orship +ac er +tur keys +quarter final +muskete ers +ðŁĴĽ ðŁĴļ +sf x +museum week +sc ala +ri sis +( ðŁĵ· +ãĢ Ĥ +z ies +bo eh +hu es +lu sci +dol a +impeach trump +roo d +don caster +tor re +hero es +fo yer +tar i +blur red +ke w +frank ly +dro id +ap al +Ð ¼ +y af +bre t +par agu +cac ao +ðŁĻĮ ðŁı¾ +ru e +head aches +shaw ty +char ley +pal er +go wns +correc tional +ðŁĺ© ðŁĺ© +breaking bad +ol ing +da p +endeav our +cit adel +tra d +incumb ent +medit ate +foo ted +ðŁĴ µ +shab bat +dayof the +wil lem +gal way +to red +marri age +f illion +sleeve less +aud itor +jin young +invin cible +kad una +a and +volcan oes +mon eti +indie gogo +buccane ers +ðŁijī ðŁı½ +ãĢ Ĥ +lay ton +cuck oo +hu mber +buzz er +Ï ī +to re +stra ins +sto m +pa ine +s we +du ff +z ou +si mi +li pp +ur n +se agu +ðŁĶ ® +sun dae +hi c +ðŁĺ ¨ +bull pen +u per +flyo ver +al dridge +glo bes +ali es +ken zie +ge es +y cle +sp lin +mag enta +j ha +bal u +gh orn +ti pper +wick er +taste of +con clave +ch ale +inv asi +cat er +dio xide +me gab +win n +at p +transform ative +nest led +hi g +bri dging +lil ies +chee red +bad dest +sc rolls +real is +dipl o +ðŁĶ « +conce ssion +prefe rences +explo des +er gon +introduc tory +ine au +ch af +som es +land rover +spir ation +sex y +sco recard +illustr ates +soul mate +wi en +inter disciplinary +fore casting +ent ities +glu ed +en lar +cur t +percep tions +boot leg +mi re +asho k +v az +hor ne +cal le +ac ulture +ther oy +night time +oc al +character design +ar mist +ðŁĺı ðŁĺı +yah oo +ac eae +to se +even to +sou t +nay anth +wh om +v are +ri gging +gen us +hi ve +com mands +sti e +day a +ethan ol +en f +hi fi +flu ence +cle mson +re invent +thermom eter +humor ous +emer ging +aci ón +ðŁĺĺ ðŁĺį +s ity +haw ke +accompan ying +t ility +ðŁĺ ª +re cess +protag onist +l ery +dun dal +int l +britt any +q bs +off the +marri ages +how to +viol ated +adel aide +wit t +lanc er +pak v +hu me +st ade +bra gging +ou tright +ad c +super st +real time +cu res +garden ers +ero ck +dale jr +ver o +bar tol +mo ti +mc fly +v pn +st ink +over rated +guer ra +e tis +ath ome +twd family +th ab +tn x +rafa el +family travel +x ley +sat anic +equ ations +ru dy +wal dorf +stan i +tu be +meas les +zimmer man +obli gations +i ously +bow ser +trans former +sho ppe +shak en +gh ouse +to d +ke tball +share holder +mar ca +kp mg +ak an +given chy +coast al +au th +roller coaster +mar ches +coordin ate +cine ma +apprentic es +par lor +mit o +men on +consider able +bar re +glo ss +enh ances +jaz eera +fal mouth +thra sh +stat en +k zn +eng el +samanth ap +flo ppy +sal om +ðŁıĨ ðŁıĨ +w ack +deliber ate +osc ill +herit ag +du sted +orni thology +pad dle +fer ns +bar un +cl ans +anticip ate +a ay +mat ically +é ĩ +tu mble +post man +unic ef +tro tter +op d +leaf let +ge ist +cease fire +scre ws +cre ation +wal nuts +longh orns +under statement +ab b +proxim ity +na x +un ity +turn pike +orda ined +dub step +chak ra +me ch +love her +look alike +donne in +vir on +Ù Ī +bang ers +vari ants +out dated +in ta +cri sto +sp elt +food and +f on +stefan i +margin al +hu tton +ti ara +tel ford +qu en +fair grounds +que tta +mikha il +heal er +v ball +ty re +under grad +gl end +hom ers +scri bed +main tains +po che +mis sal +mar ko +u as +á n +sh p +con vey +pad re +sab a +pu glia +madhu ri +pa xton +chap lain +n ago +ca si +... !!! +fli rt +sal eh +k are +di re +stam ped +extre me +ðŁĺĥ ðŁĺĥ +ho ppy +guadalu pe +advant aged +eu char +p low +un n +mac qu +port land +cla sh +pe s +lou bout +y p +keep ing +arca dia +fran kie +fi u +de th +encyclo pedia +si ze +inve sts +ðŁį © +geo logical +fran ç +con front +ðŁĺ ¥ +d ys +af m +tex an +graph ene +repost app +ac f +ur sula +gaz a +dd led +fu m +wsb tv +m be +fron tiers +chrono graph +ke s +inter faith +tab oo +spar ta +won do +flori st +em braces +ca w +no el +arch ers +ðŁIJ · +roman o +ban an +sh akers +melo dies +geo thermal +se phora +ìļ ° +оР´ +pro c +hand shake +pan de +popul ated +slow down +hor tons +registr ations +un deni +lan ts +pas sover +thak ur +li ef +adhe sive +pe tal +micro scopy +memph is +confir ming +air drop +mesm er +perce ived +ming le +lifel ine +gh j +worcester shire +pas sions +ach er +el lar +ah o +firen ze +bar ang +letter man +hat field +lu cha +je ter +e shop +william s +horo scope +pre de +east bourne +dur ga +di version +al trin +seis mic +premi osm +nar co +ti r +ori g +or m +land fall +ci ous +lin do +max ine +x ico +tra y +os wald +c ba +ric otta +n cr +mar au +ภ² +gladi ator +ch ery +lun g +u me +po psic +lon ging +can als +ta ya +decentr alized +sho pp +pres sures +mahar aj +eti had +wal greens +succe ssion +sign aling +li g +staf fer +north korea +def ying +as ma +de g +peri meter +oak ville +m sk +balti more +rece ip +de ple +ðŁĺŃ ðŁĺĤ +jambo ree +> .< +rsp b +puni sher +consider ably +in tothe +pari sian +acceler ated +polye ster +low es +fr ying +sauté ed +mou ths +seychel les +ra x +go dis +dak ota +house wives +the me +mat inee +black bird +ye sung +pre fers +pelle gr +in ated +trun ks +stronger together +re pet +re pairing +ped als +toler ant +her r +dun ne +indic ation +decat ur +b tv +exhibit ors +ik on +friday motivation +bra gg +live tweet +al ves +womens art +foreig ners +wal lets +min dy +lan ey +bb in +tv miaw +lif ter +tar get +tam e +dr ou +astro photography +mp c +g pu +nord strom +fric tion +run off +lov able +sp nfamily +ext ingui +bloo dy +sch el +arti stry +sw ish +scar ce +ph ils +max im +pos sum +com promised +sty li +sc fc +is sa +birmin gham +sket ched +angel ica +ordin ance +je ts +conqu er +ðŁĺ IJ +online shopping +s ori +reason ably +nue stro +ar turo +ch l +benef ici +spho to +wel t +ni kk +ðŁ¤ ŀ +dan ao +for mid +as se +af irst +âľ Ĥ +gil lette +as sor +an onym +sel ca +fe mi +bear able +y and +ar mory +cre pe +celtic fc +bra vo +in expensive +de lec +ge cko +new market +snow flakes +kab ir +con tra +can ning +mor pho +gar wal +ðŁĴĥ ðŁı» +fight ing +mu tation +woo dy +ju gg +gr aces +premiosm tvmiaw +kenne dy +gu p +sa e +op ha +off spring +fini sher +bet ts +span ning +mar j +h one +sh ing +contin ents +samanthap rabhu +un related +l acy +explo sions +benjam in +sophi e +no ting +micro soft +as sen +a hoy +i ker +ho fer +mo e +ah madi +yan n +an ak +ma hi +be u +aha h +creep er +baahu bali +am at +pri ory +haw keye +deloit te +sko da +print making +assemb ling +mirac ulous +no ch +sw o +leg a +oper ates +border lands +eli e +stron gh +rep tiles +pir ate +un fold + ¯ +qual comm +un predictable +ot r +rose wood +direc tional +counsel ors +corn ell +liber ated +j ad +ir regular +bulgar ian +high ness +vodaf one +sw ild +mini mize +gra zie +๠ĩ +r stats +stre ep +ome tric +humb le +lu mp +l ille +b ü +home depot +tripad visor +ki wan +a via +er z +ex ico +du f +blu men +mi zing +ar ma +in im +con stan +sor a +ju al +au n +tw ell +tren ches +her a +r k +po plar +recipe oftheday +ll an +bhu ban +short ages +ing don +bridge water +ðŁIJ ĺ +fortn ite +cam den +un cture +pro w +colon ies +t ks +n go +b hm +live pd +spl ace +sli ke +happye aster +ter rence +revol ver +j ed +yy yy +office of +m ts +exist ential +r ourke +explore bc +sse d +pri est +vix en +si ding +k pa +a har +ju ic +ob struc +foren sics +uk mfg +cancell ation +we ary +ab q +ele c +pri zed +deb ts +me zz +salv atore +m dc +gre tte +c gc +th on +snow storm +ts ch +cook ery +å ¹ +wa xing +n acional +mur s +ra ve +cap es +ger main +dri pping +sub mitting +ome lette +iter ation +aj es +shim mer +fu eling +ðŁĩ§ ðŁĩª +li po +bo bble +un follow +islam ist +hi ber +cat s +agentsof shield +sen si +____ _ +ster ia +inst al +ausp icious +har row +over land +femini sts +inst ant +char iot +blind ness +sp ed +sc arec +nu it +mini atures +ho seok +glo ck +fifa worldcup +e te +dis m +we iner +ex foli +ear ts +ภĶ +my art +man il +iss ant +form a +in cu +buffal ob +in tim +mc cul +anj ali +po po +un doub +hil a +fun gal +thank ful +fu tur +en dish +ren ds +th ar +she ff +ring o +nichol ls +io wa +po tom +cl ams +ãģ Ħ +acon f +stadi ums +di mp +di k +residen ces +do v +caric ature +seagu ll +kl m +confe ss +sla pped +cele b +turb ines +pp v +nur ture +el ab +.... .# +tu ff +de press +al far +amii bo +di spon +e wing +que er +friend s +for re +âĺ ¼ +sw t +aqu arius +head liner +cur d +fi gs +o tters +love fl +kare em +go vegan +fri yay +consol ation +at ri +ì§ Ħ +âĺĿ ï¸ı +poly ne +gu ed +o ya +la us +intestin al +cam illa +scal p +pi r +leed s +horri fying +bore tum +dand elion +fer rer +ell ic +as x +so ren +re loaded +ale ague +navig ator +ine tte +add ams +al chemist +ak shay +dystop ian +awe c +n aya +al isa +ai led +ag or +avi ator +ali zer +smo bile +findyour park +cop ying +to ddy +sh ti +mon ger +cal houn +nap kin +break up +y atra +se thu +ric hi +eras mus +fer ry +am ore +prac tise +bo bo +power point +oo se +li ffe +chin a +sh ka +fad navis +du ane +war on +fal se +ðŁļ Ĥ +wa shes +disc ip +==== ==== +g k +ab b +stub born +medi eval +p ci +ðŁį ª +maril yn +h yo +man di +cr i +prede cess +continu ation +om usic +s lat +wh al +mall ory +bon n +shen zhen +ca i +âĺ ĥ +sa fest +for wards +dra wers +bla sted +sle e +mor phe +mb ta +dumb ass +ÑĦоÑĤ о +alhamdulil lah +ec lub +al beit +heal ey +ayurve da +adverti sed +cro cs +itt les +bry son +be i +nj pw +honore e +fu sed +ðŁĶ ĺ +mul tin +n aga +de parts +ko p +kin o +jhar khand +ed na +ax le +mil ton +supremac ist +marrake ch +domin ic +tran script +] [# +: ). +wo c +sur rounds +o gil +leaf lets +co well +whe w +tru de +proli fer +succe s +sports man +con dom +po che +k up +imprison ment +{ } +scram bled +å Ľ +ka ine +cell phone +metam or +con i +remn ants +ee z +down pour +afterno on +exerc ising +ber ser +architec ture +wick low +m ns +is p +bo c +n iss +mn wild +stu mble +r si +lu ffy +sil en +dd ad +bul lies +haw ker +bb cc +scu ba +e pp +que ts +for aging +pal let +ha di +cinemato grapher +cat chers +to aster +k hi +lite coin +kid lit +amher st +maur icio +ip ad +mar malade +fe y +don nelly +g to +est as +cere bral +ant grasso +zz led +vir gil +swa pped +ðŁĺħ ðŁĺħ +no dapl +greate st +nhl bruins +fra ser +b mo +ane w +. âĿ¤ï¸ı +se gregation +remark ably +mccor mick +lo gger +er as +contrac ting +âłĢ âłĢ +yor ks +uku lele +touch screen +de cked +ben n +south wark +ra vin +nu mis +ðŁ¤ Ļ +ru t +gre co +eth ic +red neck +ar r +t cs +ih ri +ðŁĩ« ðŁĩ· +l k +inher ited +zy k +viadu ct +marty red +hi gu +ss n +be in +street style +fer gie +bank of +æĹ ¥ +stake holder +exempl ary +cre ss +ess a +ero tica +intre pid +gom es +bra un +bethan y +bang tan +pulmon ary +m illing +doctor ate +trump russia +ठ° +s ani +bl att +pla u +depri ved +t le +ful ly +bour n +st ak +lufthan sa +kio sk +far oo +def y +bad an +ðŁĺĺ âĿ¤ï¸ı +rit z +tri sha +ran ds +middle sex +arab s +pro j +sport scenter +repe ats +iv f +bleed blue +as sure +o bs +territ orial +ele n +bever ley +ann ah +âĿ¤ï¸ıâĿ¤ï¸ı âĿ¤ï¸ıâĿ¤ï¸ı +z l +for good +science fiction +gla u +son ya +pri th +st weets +mix ers +mari o +ant elope +writing community +went z +den ham +be di +sf o +harley davidson +look book +immuno therapy +or phe +es ville +ed ged +tas k +sb ball +corro sion +kilom eters +co sting +play back +ke ke +di visi +u ter +re location +yel led +pen g +up beat +ser ve +âļ ł +hal en +stir ring +reh man +en v +schu macher +frag ment +alkal ine +sb k +resil i +share point +rol lover +tra sh +counter part +âĻ « +ob itu +à ½ +ãĤ ¹ +mul berry +ðŁİ Ĩ +auton omy +spra ying +nat l +love you +fran ki +nu k +esc ar +can teen +ali baba +de plor +mole cule +pu d +fort night +blon die +sp hin +portra yal +ta che +bu te +consi sting +freep alestine +c sp +im mort +d ns +ðŁĴ¥ ðŁĴ¥ +tour de +coo king +archi val +ga thers +bit t +b anc +pre mature +snow ball +poetry day +lou dly +fug itive +ed ay +em ra +ðŁĩ¸ ðŁĩª +sci en +node js +jur gen +je ong +band ana +un is +fox sports +v andy +pro visions +wee p +tu k +i ko +h oun +zig gy +z r +fil let +bat a +tin k +con e +we want +k ilo +hor ace +sl t +sc t +stay tuned +victor ia +umb ria +att acker +ingham shire +fright ening +no ir +fr at +con tempt +lia ison +ho i +br ink +tr ill +ni agar +kick ass +dun das +not my +rho de +bu mble +no xi +fa g +spec tators +mancrush monday +jin ping +distr act +dais y +wal den +portra it +ar thistory +vol tron +ev el +is c +ac m +r ite +na o +de ported +swe ats +ru fus +lo bo +labor day +gam o +ihri thik +bl it +abdomin al +ãħ¤ãħ¤ ãħ¤ãħ¤ +i it +e q +bu sy +allu arjun +un disclosed +de ton +pro create +ki l +ðŁİĤ ðŁİĤ +mitch ell +ki i +inherit ance +al p +jo burg +pat rolling +compul sory +un signed +ni am +l ga +eshop suk +tr illi +ma w +appreci ating +rock ab +mañ ana +an tal +mal vern +roy o +grand prix +sut ton +go ftheday +dig i +ãħĭãħĭ ãħĭãħĭ +t les +varan asi +erec ted +discip les +cont act +ðŁĺ µ +li d +⬠ĩ +scen tre +radi ator +ing tips +trans itions +thursday motivation +chem ical +separ ati +sal is +mi m +geo graphical +book fest +/ . +âľ ĭ +v ae +cur rie +ag garwal +acceler ation +the ses +lg m +u mass +pro portions +nat a +ani ans +ku ch +be acons +ap r +@ # +ðŁĴª ðŁı¾ +nu ke +sher aton +ki o +ma kati +polit ico +mor ale +ì Ļ +econom ically +gg ly +ss en +pa stries +intern ships +vic ente +fanta ken +aveng ers +accu se +slee pover +indic ated +the dream +ster one +ren ders +fro st +ou i +gre gg +d ore +⾨ ⾨⾨ +pu gs +sat y +nu mb +hems worth +tam i +la ssic +schi ff +igle sias +ag awa +] " +re shi +game stop +divor ced +theat er +clau di +un conventional +prophe ts +ac in +twel f +tow ering +t ml +sc lerosis +k wan +ge ts +distur b +na ira +ener g +pir acy +pru itt +noti fied +hen na +bra m +ground water +bl s +opti mis +$ ) +luci e +biz hour +fang irling +gr ills +or l +ver se +c ina +law less +artistson twitter +tele vised +marshmal lows +radio head +bar r +m fc +bre vi +mmor pg +g aya +âĸ « +sub titles +j t +disney land +to bago +nh m +groo ve +fi awec +" / +ba o +scra bble +om ni +ff l +um c +si mba +ali er +ter rell +plu me +mi di +dig nit +co c +bru t +ad ata +alche my +d sm +ðŁĺĨ ðŁĺĨ +win try +spa res +cu er +conclu sions +to ys +od or +fl ann +gar vey +scrip tions +inspec tions +cat ap +ang lo +st louis +heim er +at ay +tr ich +en yc +chil ds +vent il +mont p +guiller mo +circu lare +z ell +mode led +craf tsman +al ina +stimul ation +cashe w +ju das +best of +to ire +susp ends +scol lege +real ising +by tes +bloo ds +as si +ðŁĴ ¿ +o hs +ðŁį ĭ +scallo p +ठµ +gi fting +camo gie +wil kes +o zzy +ðŁ¤ ¤ +ver onic +sav oy +deme tri +baby girl +ðŁĺį ðŁĺŃ +so x +cly de +induc tee +count down +self care +ठľ +vi ka +tor re +phd chat +pe ars +aw h +suff rage +le sn +admir ation +mp p +shark week +schul z +santor ini +clo ver +( * +stras bourg +ex iting +so yu +finger print +che a +ãĢ ľ +vin dic +song writers +so a +prou der +nam a += )) +simple st +delici ously +gil les +u q +mn wx +ep p +sh un +ken nel +fall on +ðŁIJ £ +sin d +tra gically +out es +modern ism +co ke +gy n +spi on +âĺ¹ ï¸ı +le am +compress or +apolog ise +twent yon +fan atics +âĻ » +sco tsman +sa wa +ko u +as er +ภļ +welter weight +phen om +twick enham +stri a +p out +ka z +gi am +cd p +ho y +emplo y +red mond +ภĦภ+sm ere +trance family +proto cols +pie ce +lu iz +iter acy +carl s +united states +har med +phd life +ch aw +foot prints +l é +cho ker +z ana +sli pper +eric sson +insul ting +articho ke +advis ing +acquis itions +op or +mut ations +re ar +ॠģ +pod cast +wi ther +kun g +íĺ ¸ +win slow +di apers +ðŁĵ¸ @ +ec ker +col lar +hu ey +gi ro +mono gram +kas ich +si veness +malay si +arom atic +gre s +gali leo +u ji +rob b +dr m +none theless +as a +: > +lo a +l np +at work +ag t +laksh mi +pipel ines +id al +stre l +re all +chain z +stone wall +san sk +ðŁı ´ +pied mont +hoste ss +ci u +t é +analy ses +wil helm +scott y +rw by +mosqu it +use mb +qu ins +ðŁij İ +tu cker +s conf +speci fications +psychi atry +broo kes +s ils +ol af +de to +co di +cli p +fil th +womancrush wednesday +go to +ang erous +be ale +w tc +paneli st +ne x +lar sen +emili o +tab leau +h itters +conce ived +americ ani +or tega +mar di +Ñ ĥ +pain tball +thir sty +new yorker +etis ation +go ss +we aker +u gh +tro ll +har ga +du al +ght ning +at ine +ðŁĺİ ðŁĺİðŁĺİ +cook out +pyrene es +po ss +authent ication +sports wear +yun ho +kir o +archi pel +shen ko +ren der +nov ation +divin ity +ðŁij £ +su fi +humb ling +ge opol +devote es +wait ress +tr ough +py ro +i ba +bl ing +gra f +epilo ts +bt r +of tball +bas king +domin os +so om +r ath +sher yl +qu el +astronom ical +wel d +track list +sig nee +slee pless +com man +ch ron +summ on +pure michigan +cri spr +sli p +la gi +ra q +um u +thal ap +char med +scru mp +quad copter +ski p +peter sen +mun i +ðŁĮ ¾ +mon aghan +tra ys +ick ed +canad aday +te gr +ï¿ ½ +hot ness +heavy metal +ab ar +gop debate +az ul +spider man +sun flowers +ľ ë +web comics +bar d +Ð ² +nichol as +slu sh +ram an +mark ham +ffici al +ff ler +íĬ ¸ +ple ss +anush ka +to to +sk aters +pro wrestling +compet es +ay ala +myster y +thr ills +mp g +independ ently +y ul +imper ative +formid able +tire less +st acking +ton gues +mal tese +pot ts +mat ti +char ting +chill out +super nova +ome o +sky sports +nu tty +ðŁĹĵ ï¸ı +ro han +insp ired +concier ge +ser ra +ma kk +gal at +chi pp +ye v +ì £ +reim bur +op ul +kimber ley +i eee +bre men +ch itec +or in +nak u +bon kers +foo ty +emer gence +ðŁĨ ĺ +sti p +serge i +zo ey +ai me +wou ld +dy es +destin y +vinai grette +dri er +circulare conomy +an archi +ss r +sch el +cin er +gro om +determin ing +gar min +cal ais +incarcer ation +bu kit +no i +chelms ford +mckin ley +chi pped +belong ed +tu mors +str oud +mi i +influen za +wwen xt +tun dra +tele communications +cat sofinstagram +t ages +beat ty +o du +ml kday +oo per +dang le +ak ley +cru mb +anti gua +ti mbers +rou hani +ðŁĴª ðŁĴªðŁĴª +ha fi +... !! +w cs +coo p +sn c +lit res +ãĢ Ĭ +ha z +co z +k ant +green field +cur ti +y ale +flye agles +what soever +wor thing +rou lette +flyeagles fly +un da +a inted +stand ing +lusci ous +h pc +effic acy +ash land +me ghan +ky wx +n pr +bath tub +ac os +h ani +mar cor +man tis +da isi +bo ba +ab bie +mu til +vi al +spy der +po z +g ti +el fie +nigh tw +metro id +anton i +mad die +dh ry +dar lings +ten ds +taek wondo +atlan ta +me ow +chlo e +ãĥ İ +ym es +siber ia +k con +gu es +mar iner +fac il +azz le +[ ... +han nover +bav aria +vir go +te uk +u sps +) # +wall a +sam pson +need less +ver bally +hay ley +bow led +pi us +lam pard +ham string +vol vo +road safety +cho king +sor bet +a hem +healthy food +brai ded +horticul ture +cr ative +che ek +ad do +the force +ko ko +schiz oph +j ie +w ada +twentyon epilots +h bcu +pro ton +pau ls +lou isa +lat am +kyr gy +com pac +sd k +sap i +?? ? +liber alism +ep silon +ai den +w usa +spra yed +baske tball +kim ono +blue wave +ali as +ë§ Ī +mug shot +ce c +do gre +ad ora +ðŁĵ· @ +kra kow +intrigu ed +exhau sting +astron omer +ven ison +lady bug +ci v +bra e +us m +bri be +acup uncture +pembro ke +ke ating +chi e +y ad +t si +sm i +see ding +gate shead +lis boa +gy p +canv ass +ðŁĶ´ âļªï¸ı +op i +ni r +soci etal +ly te +ati es +c sm +ar tery +al in +aka poor +abstr acts +âĢ¦ âĢ¦ +teen wolf +ne we +travel gram +sentim ental +per ched +han del +ho ek +f ay +coordin ating +anim ate +man ian +effor t +jer ky +f ck +adri enne +ma bly +tra ding +my el +spi ro +sol a +stor ing +over drive +monday morning +dream team +pul se +bon di +ber nie +pgat our +tri poli +son am +plat t +âļ ¡ +ag roup +îIJ Ĵ +inv ading +v cu +k ell +ñ os +un dead +pod casting +mercede sam +mana fort +cor tex +que so +impecc able +pal mer +wil doz +sport sc +guacam ole +dispen ser +cate gori +stun ts +per il +invit ations +dune din +xi e +achi eves +saf er +pre ds +ph an +knuck les +k ak +igno res +lovemy job +aru ba +ound ation +datac enter +co vert +gr ing +cou ple +ا ر +vol i +mc cle +arti sans +lu do +kal am +arom a +under taker +hu la +wiz kid +gu mb +god frey +bakers field +ker n +engine er +car ve +pal in +guaran tees +pe bbles +b ays +zi eg +fin k +â¬ĩï¸ı â¬ĩï¸ı +down pours +ro chelle +rasp berry +ðŁĺ ® +gra phies +stom p +caf es +ari zed +utt ar +cal vary +dri e +crusad er +bus an +tux edo +si u +seam us +cul tured +blan chard +town house +ge red +butter milk +flu ctu +roger federer +hel i +ðŁ¦ ĥ +u ous +ram esh +mu ppets +email marketing +ye ss +br ice +ri zio +pel o +donnein arte +u rable +inve stin +bump ing +raji v +sav a +thro wer +fore x +o hhhh +th rust +pull man +r fid +sep sis +le ed +fri ght +roun ding +ne b +ph ins +ai sha +utili zing +squ ats +gold smith +j ic +bo ks +vau s +i po +exclu sion +tari ff +po kes +min al +land s +en force +washington dc +or char +g x +mar ys +ey our +aussi e +bak ers +un popular +latin os +lar ge +pu tnam +bol o +wa de +pel o +di zz +ob struction +fla ppy +weare the +depend ence +pajam a +e te +y ann +e wan +disc la +a ay +kar ina +e ic +an trim +w soc +neg atively +kai do +fotogra fia +dh ru +colo ssal +mcle od +k wang +mani pu +ex hilar +us atoday +summer slam +co les +tapro om +unbeat able +de ma +tic ks +k ling +fil s +campaig ners +ภķ +brew ster +audu bon +qu ay +ch s +ki gali +d ler +strength ens +som al +sign ingday +gol ds +pig ment +orche stral +g q +lin kin +ðŁı ĩ +ta w +algar ve +ho v +ear le +gold fish +am ig +ex er +ben in +dru id +ðŁIJ ¸ +she m +quat tro +mer cen +men te +incorpor ating +bon anza +state fair +en de +concep tions +e es +âĻ¥ï¸ı âĻ¥ï¸ı +d son +fire arm +orb ital +we h +multi p +fo b +requi em +p light +thou se +sa id +oc re +remem brance +n old +chi pping +be v +er t +ca thy +sy m +ri ggs +m ley +dialo gues +sl ender +how l +gau teng +wd w +to bi +smo kes +im plo +b pm +ad n +mom basa +cap sul +bloom field +artic ul +cle o +goog led +flu ffy +l ard +en zyme +ve sti +ibra hi +fl ame +e mea +out ages +dispro por +ble ak +an sel +ick er +st louis +stock market +good friday +sau lt +stal led +pro m +ep som +b é +the se +sau ces +me w +lit fest +pre d +re u +kar ak +si enna +ell in +bio technology +ï¸ıâĥ£ - +tac tic +sa in +por k +mon za +ka j +lu sh +compart ment +chang ing +shraddha kapoor +fo al +ar tem +cu ando +can ola +ori ente +me sse +d ited +br c +box er +bbc two +s st +ment day +em ing +de wey +kof i +âŀĸâŀĸ âŀĸâŀĸ +reali zation +smo l +tw ood +san je +flag staff +ber wick +cor set +can ary +whistle blower +et ched +com posing +squee zed +bow er +auto desk +ne h +mathi eu +ba ja +Å Ĥ +hy dra +da im +am eri +insi sted +mer lot +gar ros +heart news +gaine sville +cut ler +bo de +ðŁĺī ðŁĺī +lew es +scoun try +g sa +us u +cc m +god awgs +phara oh +cra e +mor ley +hyp noti +f ades +neur ons +fu zz +ing co +high landers +star k +vig ne +pac kets +amar illo +reu ben +insul ts +bas ic +vec tor +n me +ac ruz +tro s +transm itter +ðŁĺ ŀ +interpre t +ðŁĺ ² +pre quel +mc gowan +dis semin +ðŁĴĺ ðŁĴĺ +mascul inity +indie gamedev +ali ve +te t +pe tal +ema iled +ar med +ko o +he er +ba ird +super junior +metro polis +delav in +decl ines +stit utes +Û ģ +p tbo +g lan +cho res +e aling +chri ssy +ste mc +vi an +assassin ated +pron ounce +illeg als +discover y +cav ill +fri fotos +f al +so i +sabot age +t int +p dc +ðŁİīðŁİ Ī +ãĤ Ĭãģ +ji o +endeav or +in sig +commit tees +she arer +me tz +mar rying +h dd +g by +fre t +tri sh +pu l +scrip ted +sa ki +l w +ke ye +shim i +nan aimo +ca h +à « +tem pered +ici an +du gg +dish washer +air field +s rugby +gr inch +y st +r ms +mahat ma +lan kan +disc ar +dige stion +no des +l ls +om ic +gu tter +tis garh +feder ico +election day +bo he +master card +fire ball +âľ Ķï¸ı +oy ster +p ong +do k +en route +m vc +beat the +ali stair +shu b +sh aming +cherno byl +ghi bli +the s +pin ion +d bs +sal ts +ic tion +epi ph +nc pol +in convenience +whit ley +inspec ting +wood ley +wi ener +skil let +no les +m ca +h ina +a sha +willing ness +well ness +tam ed +show time +dis advantaged +ber nat +us n +mission aries +coun selling +arrog ant +quant itative +leg alization +ho dge +energye fficiency +cameron dallas +pos sessions +p bb +harris burg +v g +hindu ism +happy thanksgiving +fi b +re acting +tweeta picture +pol iti +mu ppet +hur rah +pac e +coast guard +guar ded +as am +par ry +fore very +x q +oom f +ke anu +j ind +ri st +customer service +sac red +ðŁĺ º +ton er +occur rence +mat u +val dez +red d +is ak +power rangers +pe asant +raj ini +abra ham +e mil +car do +tr il +hair styles +obsole te +sam pler +direc tive +delavin kisses +ver ton +glo s +sp ay +paler mo +com ets +man ziel +chicag of +ski pped +pic torial +h ant +b mi +a ol +re opens +pad dling +devo s +fra ud +bas eline +que ues +sp ired +sn are +eu ve +descri ptions +daisi es +ca ching +gall eria +tri mmed +stin o +recy cla +ic ular +bir ken +raw lings +fli x +chic as +b gt +lik eli +argy ll +thel ove +ga ston +bl anca +ha k +f one +sailor moon +h aci +ima c +fl yn +de can +bel les +ap ic +zo g +taun ton +con stance +lasag na +ker nel +in ka +har bor +collec tively +calcul ated +av ille +shil pa +pur du +gi mm +fun er +a est +pembroke shire +nighting ale +n unes +hyper tension +hu bert +sli ders +infer tility +comm ended +transat lantic +metr ical +!! @ +Å Ł +ss g +bac ca +inver ted +fun factfriday +it ans +albu m +acqu ainted +ri er +whel an +sar ab +mu e +snoo ze +pi ff +agre eing +sp itting +jer maine +n ye +âľı ï¸ı +am bush +ze ph +con greg +univers ity +s app +wann abe +pat rice +ib d +do glo +fri dges +sun d +king ston +ar gon +kam en +hardro ck +ds ley +do lores +ì ° +ota ku +pi ping +be having +âŃIJï¸ıâŃIJï¸ı âŃIJï¸ı +blue bird +an sari +teapo t +fire work +cro p +log ans +ty ped +thick ness +ig ers +c fp +dys functional +contra sting +et ty +aston martin +tx st +dra grace +at tributes +marath on +manu scripts +john stone +ðŁĺ± ðŁĺ± +bo er +ay u +aru gula +poo rest +con du +assu mption +anag h +no h +delav in +sit ter +g ö +mor ow +kick start +com i +gl acial +ghe ad +ba in +ker shaw +en dof +fre ud +om at +i af +hu g +sign up +each other +defin ite +tu bing +shak ira +ðŁijı ðŁı½ +uu uu +sw in +sham bles +ol as +sk ell +brit ain +kn w +clu tter +om y +j ens +hang ed +city scape +scra ps +un locking +dead liest +er no +breast cancer +a it +inspec t +fu ri +ðŁĴ Į +ku d +ju le +or ah +mi ds +m dt +bur gring +r attle +pu sa +stal k +cle ans +iss ance +z ek +worth it +nam eis +musko ka +council man +urban art +bar rac +un solved +tu l +g ita +white board +soy beans +em ent +cont i +saturday motivation +conveni ently +doc king +t ado +âı © +sp ino +puppy love +po f +fabric ated +robb ers +adop ts +ti fied +kk r +indulg ence +notic eable +macqu arie +chap el +sensu al +ki ko +melan oma +lore tta +li ance +ab en +sp lus +ga al +ac ele +lib dems +compar isons +ðŁĮ µ +rhy thms +mer y +en capsul +nap ier +ðŁijĮ ðŁijĮðŁijĮ +ðŁij IJ +plat z +fre sno +re formed +ran bir +el it +the best +bhu shan +vin nie +impro vised +s ittin +re created +e ba +ec ker +ac rob +pon te +cor d +gi ddy +eur usd +fe ver +intu ition +gar i +dum mies +bud weiser +amend ments +te tra +sch nit +ay as +mar ys +ci st +k ani +ker mit +ðŁĺ±ðŁĺ± ðŁĺ± +tin ker +strol ling +di visional +niger i +omin ous +menstru al +kar ab +k hy +bw fc +pan handle +l illi +well er +stra pped +son the +transfer ring +ethe real +sne aks +ru dol +gab les +jac king +cin code +for tune +canadi ens +con for +ab normal +frank lin +tit a +mu la +persi st +cu ties +ki el +ðŁĩ± ðŁĩ +her mann +aw k +fi asco +ko to +we ta +hi ker +budd y +preven tive +mcgra w +game boy +forsy th +top shop +si ob +sad h +in tram +follow art +so aps +dragon ball +ou x +morri son +๠ĥ +lu bric +adul thood +morri sons +âļ łï¸ı +her mo +ta ka +stall one +mis use +team gb +ra gha +con fined +at y +hom ophobic +nw o +sky news +ho ya +ac rosse +wi iu +pur ée +jed dah +ðŁ¤ § +advis ers +ph ine +an is +scrump tious +ë° ķ +c ke +vin y +ter m +s dc +o do +home school +vas c +leop ards +debor ah +illic it +cur ran +as roma +nau ght +mar ig +brand i +em p +ðŁĺį ðŁijĮ +î Į +su spend +lu z +initi ation +sch aft +jensen ackles +craw ler +post doc +des ks +trail blazer +den omin +tri x +no ise +po et +± ï¸ı +s mug +vol atile +proof s +pharmac ist +sardin ia +mash able +kim chi +co ed +schal ke +doo dled +c sw +sh ur +ro x +do k +chris brown +mathemat ician +ab ound +ang elic +rock ford +d ole +yor kers +ms n +g man +xavi er +bor rowing +mark ings +longh orn +k ja +diver ted +mm it +euph oria +ay yy +te a +pa h +ck i +un cut +li ven +ky ung +fan art +mer ing +red ding +amo vie +gri di +c thulhu +schol arly +ju dah +th bewithyou +eu calyp +ðŁIJ ķ +hert fordshire +cour troom +by u +auc tioned +ple ase +mar cia +ê° ĵ +succe eded +el as +arvin d +t lot +saig on +re tt +ra kesh +fd ny +as en +se bring +gladi ators +you know +v lad +gol a +par ap +ÑĢ и +sab cnews +one team +oh l +sun e +ri j +cd c +star gate +run down +plat o +ph c +chat ter +ra viol +mn f +mand ala +li et +ภķ +mari a +hun gover +consoli dation +fer rell +tradition al +ilove art +gal ap +ðŁı Į +que zon +espa ña +ðŁĩ¨ðŁĩ Ń +ho bby +steam boat +mali gn +guil lau +pro hi +its me +íĥ Ģ +in scription +al z +mari an +k ade +mm on +adju sting +ne sts +intern ally +ci r +vik ram +mal ala +k ph +fel icia +the real +cap tivity +at is +marcor ubio +kale ido +che v +mano j +le more +gent ri +vi ps +tro pe +" âĢĶ +pair ings +mal nutrition +fr ay +desig nation +brun omars +az e +tor rential +pan zer +ga il +under the +the ological +schizoph re +dazz le +freder ic +mo par +ad illa +so ggy +ra un +medi ocre +colo rec +i fe +p inst +blu ef + ² +world water +gir oud +clar inet +ad olf +tar antino +receip ts +assu mp +ðŁij Ł +coffe es +âľĬ ðŁı¾ +du plex +s of +r x +lin o +timber wolves +pan dit +mo tm +e ga +ay ama +ach s +outsi der +ll en +co er +til ly +cheese burger +ma ds +ple dis +emp ty +national parks +az iz +p mi +jun kies +f ener +sq n +è s +gener ation +cleop atra +bhuban es +mosqu es +ty free +popp ins +tw c +or well +n age +ka whi +hol low +dal ai +¨¨ ¨¨ +ou ro +m health +gi on +az o +vis as +reneg ade +re ic +w sop +ðŁĴļ ðŁĴĽ +e chel +tox icity +mü n +bun k +stimul ating +asth our +\ ' +ep h +ende mic +cn bc +shrin king +peabo dy +michel angelo +can yon +wal e +su mi +si ders +inu it +? . +profession alism +dr acing +plat oon +p ons +out bound +maple leafs +de sol +cen cy +a than +ver ma +ru bbing +ok an +ðŁij ł +mull ins +authent ic +Å į +alman ac +ga ia +bb q +on imo +ke h +ty a +tou ts +y av +re posit +, . +wi ght +se eyou +cal lof +done sia +bar gaining +gr anth +sd su +amphi theater +p su +re watching +wine tasting +peak district +dete cting +thur man +phe e +èª ķ +u mich +re r +sculp ted +go le +name sake +ðŁĶ ģ +serv icing +bau gh +pu gh +pen cil +dar th +munch kin +at orium +ten ers +sun y +rolling stones +mag ing +star rer +i dris +fe instein +ag ron +âĺºï¸ı âĺºï¸ı +supervis ed +chamele on +aggre gate +succe ssive +mo gul +inst yle +pol dark +custom e +ohio state +ha ya +ci des +broker age +angel ou +fifa wwc +de forestation +al ton +pam ph +hu gged +ho bo +change able +ku ber +bur roughs +demon etisation +cape cod +vers atility +or ice +le ila +womenin science +tu a +he dges +embarrass ment +ali fe +so ars +ni ghter +hy mn +gi pp +chas u +tech s +ni all +k illa +hi ka +cam els +valu e + ¢ +sc oops +mah moud +clu sive +adri ana +pac o +oz il +un as +transl ations +whispe rer +s bi +bu xton +bio tics +indi ffe +ken ney +k lar +et ching +barra best +inst ability +se ine +vo tel +blo gged +whis key +my space +t ant +lan dia +give back +illu s +aw ak +ac ab +f bloggers +cloud computing +blat ant +syri ans +band ra +sty n +an em +ke ted +kar thik +barun sob +pin ot +gu bernat +gay e +arti ste +i fied +conven tions +hu an +geni uses +eeee ee +fol ly +somer ville +pride month +ðŁĩºðŁĩ¸ ðŁĩºðŁĩ¸ +chemo therapy +paul s +bak ar +ìĦ¸ë¸ IJ +taiwan ese +fol lo +c ss +re ign +nn nn +fla un +catastro phe +iti es +frag ments +extre mists +ym oun +car men +eze kiel +conne cting +se h +man ta +remodel ing +we ymouth +at oms +ce m +ne well +lu mi +the open +mo c +mili band +g land +z shq +mag gie +mani acs +m sp +ad y +cre ams +le anne +e sta +py g +af finity +pray er +dun bar +ligh troom +ac adi +wyn onna +roman tic +state dept +sick le +wh os +lam o +et our +fin ity +shru b +shar pen +pun dit +ed on +af ore +mar s +jeff ery +ter ps +medal list +kath arine +accu sing +ta z +roy d +from home +confron tation +alle gh +ðŁijī ðŁijī +refresh er +ran veer +never land +jo jo +lu crative +en am +ca ver +pa edi +man jaro +flu ids +the ssal +oppre ssed +mu ss +joh anna +Ø ® +cn g +buil dthe +sett les +s ith +fu ego +cl amp +ar ag +pay er +ted x +mand y +inter stellar +fr c +ch and +b cc +mo lo +len til +johan sson +grims by +nature lovers +ðŁļ¨ ðŁļ¨ðŁļ¨ +shin de +x in +international dayof +transiti onal +sat a +cad dy +wo d +if u +ha ys +holl yo +j ang +ir c +co im +grad able +" " +ðŁį ´ +ঠ¾ +a el +n yo +west lake +time out +sof i +phenom ena +cultiv ation +ag no +un armed +so t +con j +gen o +royal navy +nutriti on +fair mont +ti relessly +sn g +re ty +mic a +lu cent +slo ane +droo l +riz al +od ell +critici zed +. '" +la ze +deser ted +co der +pra s +l illian +itiner ary +dav y +an ap +whi pping +hobo ken +kare ena +çľ Ł +vi us +ter n +nan tucket +mis understood +bu laga +st ant +chin ook +z am +reli es +d ss +ed mond +sket chy +m ell +fe x +rec tor +dist ill +day dream +wine maker +ri pley +billion aires +hel ene +ati f +cul prit +bertr and +wou ldnt +ma pped +v ak +gla dly +parliam ent +kidlit art +ware ness +goli ath +âĨ ĵ +view point +tat ted +fu ls +dor sey +ang lers +li ds +ki ya +bow les +be h +b ite +compati bility +ance stral +pro x +beha ved +gubernat orial +ch field +sab an +z h +teen y +shibu ya +holli day +pan cy +âĿĦï¸ı âĿĦï¸ı +seun gri +? , +ðŁĩ¦ ðŁĩ· +im itation +impac tful +any i +gene vie +añ os +bate man +gli der +af ar +ra sheed +effor tless +sh war +dach sh +er un +at os +kin i +ch d +kha ki +k lin +felici dades +bel o +as l +to ppers +fin ley +stac ey +rigor ous +kar ting +le ppard +car michael +be ret +c se +ak hi +mer ingue +ab an +ha ke +ger i +er jee +re sto +comm anders +pr it +fl or +ad ven +ex termin +remain der +å IJ +es g +martin o +lulla by +| @ +mi gn +in store +big bang +cor di +cau ley +ante bellum +dg ate +cro ck +span dex +scaf folding +ore os +ê°ĵ ìĦ¸ë¸IJ +pom ona +ma uro +uni versi +re mi +af ootball +t ant +sm alls +ne h +worl do +tropic al +mor ph +jav elin +gla r +arqu itec +reminis cent +tu bs +spide y +make u +syl la +progressi ves +blo t +shor ten +keep in +ch ak +ang st +super food +decad ent +ston y +neuro logical +ar boretum +ann ak +fe ma +per cu +dis respectful +small biz +lo x +co om +c sc +bs bi +pre valence +him ss +esp an +mo ga +fr ampton +sky map +mas se +levi athan +( ). +noctur nal +car ameli +ang or +amne sia +outsi ders +she alth +rhin o +ant ag +ag io +ðŁĴ° ðŁĴ° +take me +kab addi +c si +m sh +coch rane +thessal oni +sil a +ha us +du sting +obe se +mack lemore +mani sh +len in +m dc +gro wn +shef field +s rs +ke le +car son +ch um +dah lia +can tore +opp o +how ling +cyber crime +sur realism +sc ran +fa iz +thre n +rac ists +r out +pk not +se mana +sin i +mc cull +ma chi +alfon so +y b +sar dar +kend rick +den g +reci pro +on f +doom sday +bri bery +custom iz +art is +c pi +ðŁĻĪ ðŁĻĪ +sla va +let te +en s +âĿ¤ï¸ı ðŁĺĺ +cra yon +ad an +tr c +migr ate +simp son +row ers +king sley +farmers market +shee han +ne phe +bor non +car ton +mic key +all ure +u lu +sli pknot +heb do +gui do +dog celebration +online marketing +acceler ating +) .. +origin ated +macar oni +ed tech +out field +mit z +disc us +adverti ser +man or +ha shi +descri p +cap ita +ful bright +recep tor +con n +con ey +spion age +r attle +pre st +u li +blog post +acker ay +) âĢ¦ +red velvet +mat th +inspir ing +b sd +ker ri +po con +mil lar +re pur +accent ure +ä ¹ +ram bo +ragnar ok +dele ting +british museum +pat ory +leip zig +flori an +sci fi +in ers +br ate +yo y +melis sa +ab er +ma sa +po te +mosquit oes +transpl ant +r pa +; )) +bast ille +yl an +joye ux +melo dic +cap tions +atri st +roch dale +gott i +pew die +cuties aturday +who is +aqu aculture +tiv a +sp el +he ss +ha ji +fred die +co per +brand o +v k +photo book +* , +my dayin +micha ela +brune i +sr ini +in te +Ä ± +de ol +d fc +separ ately +bun d +ve sts +to c +me ck +rein forced +constra ints +car roll +sq ft +re ver +cam per +bird man +in action +gener ators +triumph ant +pe sts +o vo +gy pt +al amo +sc aled +suresh pp +sd n +is mo +gi os +) @ +justic eleague +restaur ant +gab i +den gue +next gen +exemp li +ap ex +inspir ational +down side +kid z +u pl +et na +alvar o +fel dman +bar net +m ha +es ch +bloo ded +>>>> >>>> +kan i +ho fficial +casablanc a +bir ds +ty ga +sw amp +o day +new castle +nb ap +ci sion +cho ols +af lo +ne p +mon ton +ak b +super model +down time +th os +sc wx +snoo py +ag greg +yo ke +nor cal +we tt +prolon ged +me tast +beat er +f ta +t lap +disgu sted +y h +voice over +itch y +ip c +ðŁİ ¾ +phe asant +stra its +ram pant +j g +fer til +assu res +fortun es +sal inas +liz ards +kett le +i bs +cyn thi +he g +mc cr +soccer oos +happen ings +cor den +ðŁĺĤ ðŁijĮ +t ches +egre t +wolver ines +congratul ated +ho gg +bott ling +wr i +fer ri +bo sch +af ire +og den +s jo +j dm +sv t +con tex +tol lywood +min k +me se +super sonic +op oulos +å ¸ +âĶ ģ +knuck le +gu ise +gam i +chu cky +z inger +radi al +compla ined +bo da +fe tal +discipl ines +cor ro +ðŁĩ®ðŁĩ ¹ +op ted +filtr ation +ad nan +em cee +mi stre +insom ni +fer gus +tra jec +on don +med tech +tanger ine +madra s +gru e +cab s +z hu +sureshpp rabhu +insul ated +day swild +pp m +band ai +v day +s ff +squ id +lo thing +not dead +expre ssive +cu ll +ala stair +x u +up front +fish ers +en es +um d +dis missal +sti er +sel s +lu st +re active +prote ster +eyel ashes +al im +goo de +gre eng +da ir +com pen +anush ka +proto typing +ma pu +bear ings +ðŁIJ Ł +for me +bsbi botany +timo thy +out skirts +am bed +are tha +wend ell +stre aks +ni m +k pk +sne e +fit ter +quo ta +p ate +win ning +ðŁį Ń +sho pping +ma inst +cul ver +ste vie +mcfad den +counter parts +gren fell +fol som +dor set +tech crunch +⬠ħï¸ı +tip tuesday +us l +tre x +geor gie +ranveer official +lic ks +se wn +k f +' âĢ¦ +jap s +p ate +orth op +fe sta +stra s +mon tal +hammer smith +fore most +wido ws +mad re +ite z +mito chondri +lig ans +z ona +cari bou +m ss +andre i +weather channel +gh c +: ... +ta ft +awe ather +al isation +bru tal +bliss ful +nik ola +mal icious +q m +mpg vip +bro die +bl itz +applau d +dri bb +v ague +dog go +transl ating +interpre ted +hat ched +ge tyour +benefici aries +spar ring +caes ars +aw illiams +la hat +bro ke +ti mp +virtu es +rel ying +pie tro +k tn +ici sts +pab lo +lou i +a ag +pn pp +cha st +pul ses +fini sh +usair force +type writer +thomp son +dog s +ut to +ãģ į +sand al +new ly +do ge +z w +wan kers +ne gr +mu cha +determin es +black fish +sk unk +mu ps +instru ment +phy to +daysto go +skin ned +hai der +con ten +ðŁIJ¾ ðŁIJ¾ +we iler +undoub tedly +chair ing +wall is +sh ard +zind abad +adul t +absor ption +pre sto +deplo ying +drum mond +battle front +seag ulls +how dy +juda ism +des de +part ition +âľ Ŀ +no logy +national bestfriend +lesn ar +film fare +co asts +christen sen +ac an +mb u +co pped +ru bble +sw c +fun nier +far ther +where as +nano technology +with stand +pil low +bow ers +to pe +it ly +con fit +ma kar +comfor ts +bo sh +cli pper +bal la +sti k +mil b +safe guard +musi que +eas port +ya z +pad ded +bad er +fore ign +chop in +archi ve +o ka +tran sporting +tml talk +aj it +consequ ence +sc roo +ff o +collabor ated +pug chat +ye mi +jav ed +au burn +o of +ma w +sau cer +miti gate +i les +evangeli st +ter ie +re cl +indic tment +cat a +bright ness +may the +whim sical +un lv +key word +cu min +med way +west world +tra w +im posing +form ity +coul ter +ab z +ny pd +grass i +kel sey +qld pol +clock work +f dr +di anne +âĺ ij +ad h +p ann +bra vely +ae ge +un lawful +ver di +pocaly pse +phar o +kar la +reson ance +ma stiff +la dak +bu u +ma iled +hi i +craw ley +tor rent +mach ado +liby an +effort lessly +fal sely +q vist +ke ef +craf thour +cheri shed +val kyrie +s ari +kal amaz +be he +ðŁĮ Ļ +th im +ro ddy +col trane +but chers +ach im +wk end +awk ward +cab rera +:) ))) +fran c +decl an +con dos +a ja +pandor amusic +char ter +ph ill +mon trose +hatch back +handic app +gre aves +eucalyp tus +ut most +t son +bur ton +mid wives +in cur +ðŁĺį # +moo d +compre ssed +tom a +must ang +mo g +as ana +te stic +sho tel +in sol +cor sair +nh q +ben ny +sm ma +kap ur +in con +jon as +ener gies +don al +as ad +se z +n pa +archi ved +stimul ate +do p +hy d +gri eving +ãĥ Ī +ron a +why te +tree house +ss ell +sand ro +ko bo +ther most +se clu +hi ya +ge ez +mam as +prisc illa +flav oured +fas s +w old +maker space +cospla y +p tv +happy valentinesday +sequo ia +love craft +gu an +d tm +ci i +yoko hama +pos thum +re q +ðŁĶµ âļªï¸ı +galat asar +dol by +hamp tons +disturb ance +stone henge +ok c +disrup ting +month sary +jun gle +head lights +du stin +micro sof +happy mothersday +ko ko +gra zi +te sto +na idu +mal ay +ari al +ru mb +ab oo +har man +tra pe +spo ils +je ho +go dly +lock screen +z un +pi ous +ma gento +l enders +prob able +corpor al +m our +aw al +su a +call me +ton ne +go vin +devast ation +x j +gear box +war lock +per me +it ate +gaza underattack +du val +paras ite +clement e +le th +i va +fro zen +tho les +to bin +cair n +s ill +luc kiest +conver ts +st ale +pan cra +euro pale +wis dom +sch ur +ì ¶ +verti go +bi j +u bc +nu re +righte ousness +mt c +factor y +ver st +revers ed +hur i +hee chul +fab er +ar r +ul ous +ven om +ph at +green ery +bra dy +à ¦ +: (( +never giveup +di sha +mo ta +health care +dun ham +dex po +den zel +bb ins +f ics +wh am +mc g +eli an +wat a +str alia +tel lu +pe sky +spin off +ar moured +re acted +do fficial +te du +sag ar +mor ally +paralle led +fi os +dow ner +dau gh +re do +world cup +tari q +bar ne +glaci ers +oc cult +barbar ian +her mosa +!! !) +y ur +inter nation +p ss +sit u +p int +american air +sw am +dopp ler +ðŁĴĻ ðŁĴľ +cincode mayo +le van +hell enic +mc ne +ju di +yu h +st x +qu are +ðŁĺĤ . +sti g +g els +mot ley +hard work +euro zone +e ad +ç¥ Ń +seab ir +ci us +la id +alpac a +presu mably +pewdie pie +boo ted +am ari +tam ine +sol ace +bar row +acade mies +x ian +om ination +dun geons +b ma +de ity +ai k +stab il +hir a +affection ate +ving ne +new port +ãħĭ ãħĭ +thir ds +re tains +aroma therapy +ski er +ni ma +do pe +cr inge +con domin +to or +anim ator +sar aj +seas cape +minim alism +lake shore +calla way +berg man +à¤ Ĺ +whisp ering +stupi d +ri ghtful +requ is +ir n +se va +ut pol +tuber culo +squ ish +de but +govern mental +christ ine +all man +weap on +s ito +bur i +lo lita +leaf y +fu ch +tin ted +mck en +a hahaha +ðŁĩµðŁĩ ¹ +repe al +ne gan +ðŁķ Ĭ +tail gating +game insight +ðŁıŁ ï¸ı +yaku za +z t +ti ring +pro posing +bow lers +tra itors +ak shi +cler gy +cit o +up sets +tu scal +symph onic +sil ently +shu ff +black well +ðŁĺĤ ) +ko be +rober to +ri dg +dc u +mer ino +ft p +east side +. ~ +nb l +mn leg +ts for +frau dul +ca pping +in my +gymna st +ston es +ss in +twe aks +shag gy +oak land +dem sin +sang ria +mm va +hen nessy +down ton +ri ghtly +in it +aga ve +ob last +northe ast +friend ship +dal a +tro phy +ðŁij ½ +mag in +margar itas +ê · +ww fc +fa sh +di ke +cu d +char t +ðŁij ® +refuge es +jop lin +n cs +imp y +firm ware +pas cu +flam in +health tech +bell letstalk +w aka +ol ls +la go +co wan +bombar dier +sh ome +ðŁĻ ħ +mc master +na ve +well s +u ta +tell ers +mis fits +kap il +face off +af firm +a pro +whit epaper +super yacht +speci mens +al located +... , +- __ +ka w +dachsh und +djo ker +s work +qui ere +or um +ðŁIJ ł +som m +c mt +ingh our +skin ny +lgb ti +gi ggles +break away +resear ched +par ity +my al +ms l +re tained +si vity +make inindia +sol ves +defam ation +wal tham +sri racha +road way +concep tu +al in +iw ant +å Ī +del ft +tender loin +ga ins +faul ts +sw ire +st ellen +pol lo +dy ne +bornon thisday +asdf ghj +sq l +sali m +advis es +vo ip +ìĹij ìĨ +un touched +she il +ontari o +uph ill +so bre +de shi +nov ella +du tton +craw fish +ا٠Ĩ +ma a +tw ine +kal in +ðŁĩµðŁĩ Ń +ye ss +brook s +hoo siers +ton ka +umbrel las +ay ers +ate am +acqu iring +su ction +ä n +wi es +tari ans +soci o +mat tb +shepher ds +o so +charity tuesday +s logans +ninj as +al bat +by te +bash ir +trampol ine +mydayin la +i ja +bas el +ror y +gol die +fi rec +un noticed +pecu liar +sch a +ker son +mour ns +liquid ity +qu ipment +hi bs +ar s +aeron au +slide show +sla bs +delici ousness +sk itchen +hta fc +full erton +cre ighton +aer ob +procrastin ation +az ores +white hall +uss occer +medi ation +djoker nole +and me +um en +noxi ous +jo ss +ili fe +anni vers +sudan ese +et res +under mine +whole foods +diso be +kor i +ade le +eli z +can ti +al on +gymna sium +sarko die +meteoro logist +yl de +ste en +stamp collecting +nas al +lo tt +fran ks +ex ol +ack i +good year +animal rights +y les +vio lets +mm es +s thel +ra pping +tu scan +wai ver +tur ner +eat local +northe asthour +anim ations +tom morow +t sh +ff ame +bra e +pe tron +glam our +br yn +d cs +bal es +ðŁĶ ¶ +bro v +bre v +b ons +physi que +car ne +x e +elix ir +vol ved +l oma +ìľ ł +æ ĺ +van u +ri gs +bal ance +va res +bon ita +sprink le +perfec to +di on +le ak +calcu tta +o ba +d ma +c mon +tun er +pneu monia +bo gus +apolo ge +cl ough +bor ne +)) )) +revi ved +o varian +ner f +c legg +fan fest +cho u +reali zes +mc n +li gu +leg alize +just saying +for ster +bo sni +k hi +in dom +hei del +en cryp +si ss +ed di +mar bles +brisban e +y ing +pre paid +wal sall +cooper ate +orche str +mar isa +ho wie +che wy +bren ner +andro meda +e gan +sto cki +cav endish +ag an +ban o +de ir +go g +bl k +re thinking +ch ig +rhe u +sni p +p eng +semin ole +m swx +an nex +lyn da +lewisham ilton +cu mul +tb l +dolph in +agu ero +........ .... +pre lude +at our +gr anger +too ting +ro tun +dis ar +home items +da res +**** **** +ðŁij Ĩ +compre h +jin x +as well +iri e +circul ating +ðŁIJ ¥ +over board +cultiv ate +rhe tt +oriente ering +ca k +bal kans +s itt +jas min +britney spears +ro tor +se aling +g bc +oc ci +f as +eman cip +com er +war time +tic kle +son ny +pac es +log g +at rix +sr p +g win +do bbs +uz be +the wanted +dru sh +ex tru +m icky +honore es +dar win +re dux +mm j +ram i +jalape ño +io c +do ver +ju ju +whit ney +s eng +en ly +au ch +archipel ago +vigil ant +man gal +wil dest +parano id +hal i +bb ly +sanc tioned +real ms +con co +u ddin +c sk +play time +libr a +sav ag +oc tane +rec tan +re turn +par rish +mor rha +cc p +c mu +sa iled +se vent +ro sie +pil ing +he w +boar ded +seg ments +neph ro +( . +cr ats +bak es +ðŁį ¸ +back tothe +sibl ing +kirk land +ke o +gu wa +bre ads +ðŁĺľ ðŁĺľ +t q +haras sed +ga u +wil bur +j isoo +ep er +li sam +tri ppin +sh ino +ru kh +beast mode +cho a +inst aweather +rich land +gar i +fe z +cowboy snation +fur suit +k run +a en +sycam ore +se gun +ent ennial +di h +o ax +demsin philly +ðŁĻ Ģ +sn hl +pen nies +pass words +ma kin +ty e +d eng +kni gh +jeep life +hel pline +a for +zz zz +ste amy +pic ker +iter ate +happen ingnow +ki b +bloom berg +martyr dom +bul ly +assor tment +a hora +zo e +no i +illu stri +agar wal +p sc +electr onica +recruit er +gar diner +rad ha +naf ta +dot net +pi ero +geor g +bel s +ðŁĺĤ ðŁĺį +tuberculo sis +run nin +mor is +haul ing +ev oc +bre thren +sha ir +frame works +a stu +ri gid +ku ma +kre me +jin nah +insu rers +ny u +f ere +nol lywood +good vibes +- ... +toi le +sk ril +instaweather pro +cze ch +pa vel +one piece +nike plus +fi let +cav ity +ðŁı½ âĢįâĻĤï¸ı +ðŁİ £ +dra stic +dail ys +siam ese +re bu +oste o +lar k +f re +sh elling +p é +glad ys +ðŁıĢ ðŁıĢ +gusta ve +submer ged +grand stand +att u +won t +f pv +b ley +jon i +ang ames +weigh ted +al ou +ठ¶ +les bians +f j +anni es +am l +dor ia +dav in +be ta +can c +madewith unity +ha j +bad lands +mu l +blu ec +pa wn +cov ington +neuro logy +htt weets +dysle xia +thel ove +ne at +fork lift +autom ate +une ven +monte ss +he in +ha g +rel ics +competiti veness +can elo +mar tens +bullet proof +sk ittles +g ya +pri mo +americ afirst +woo o +abor tions +?? !! +ma che +ld ers +rl ly +preli ms +direc t +cour se +swa in +super cell +ec centric +sting ray +ple ts +wil cox +west in +okan agan +kir an +car bo +bomb ings +ra rest +bo h +gaw d +di gg +mo ana +enti rety +en closed +dodge ball +par ton +milky way +at r +thorough bred +re ally +qant as +epiph any +ine e +aero smith +spi eth +ar thro +ell ini +du bu +bra ving +âļ½ âļ½ +re structuring +illumin ate +equ ili +mp i +ash ton +pony tail +ma scots +flat tering +cru m +ast a +à® ° +stranger things +bar nab +ر ÙĬ +make shift +got cha +will am +cho irs +kilom etres +gho sh +eu than +dol ly +un ning +the ar +cre we +w sw +j ace +dis miss +ke an +ho ta +kh at +~ > +thir u +ren dez +hart man +tee ssi +cas ca +z ah +hydr ange +fo d +aw p +mzan si +thick er +nago ya +ne va +sti que +cast el +dam ian +there by +ji ang +ale k +music islife +ra q +calla han +gou ache +somal iland +sean hannity +ra heem +lo se +elo ve +whar ton +rectan gular +illustr ating +har ne +auti sma +scra pped +ell and +decre e +nag pur +ki pp +so re +n md +ma as +gun a +gart ner +bel li +then ight +je on +gendere quality +gi ver +a el +gar ments +ne u +mardi gras +mar sden +ro wer +pollu ted +camer aman +vin od +be asley +cro c +ji u +hollyo aks +anesthe sia +al les +ste ward +lati mes +ðŁĩºðŁĩ¸ðŁĩºðŁĩ¸ ðŁĩºðŁĩ¸ +tic ian +gor ia +come dic +ðŁ¤Ķ ðŁ¤ĶðŁ¤Ķ +nai ve +sli ons +ł Ī +bur glar +ðŁĺŃðŁĺŃ ðŁĺŃðŁĺŃðŁĺŃ +york shi +se ñ +fan boy +lau rel +inci dence +potom ac +rober ta +presi den +pr yor +os bourne +w ku +te me +pal ae +ðŁ¥ º +re boun +itu de +red dish +k hand +coloni alism +north carolina +ðĿ Ĵ +manne quin +lady bird +ta sty +knowledge able +g shore +ðŁĮ Į +à® © +qu aker +salz burg +med alists +chy na +bridesma id +ma ori +ro p +outra ged +in adequate +truck ers +al ana +ìĿ ¼ +ri x +oooo oooo +command ments +lam beth +aa j +eco friendly +bla z +morecam be +boun cy +rou x +rai ded +mi zed +sh c +gaw x +labor atories +ru bs +rest room +consult ations +ca jun +virgin i +so ir +rev ue +ple in +wag er +ç ¹ +we do +growing up +! ðŁĺĬ +face ted +sin ners +ho vering +ti ene +seas oning +an ja +leg go +il is +fla x +dev o +ash ram +mati sse +ker i +go wer +bo tox +mar shes +unh cr +ts m +opti mus +dun i +stu ffs +so k +order ly +n bad +islam ophobia +raviol i +fab er +cre ds +won ka +in fusion +over weight +daily news +assi mil +acol lege +medalli on +kili manjaro +sti ff +tham es +sun ken +th ard +my dubai +hilari ously +han nel +plu mber +fair view +separ ating +rasc al +qui en +necess ities +confeder ation +ll ll +: ] +weak nesses +bron co +ra ffles +el ot +ãĤ¸ ãĥ +advent calendar +ðŁİ ¹ +stra vel +tun ic +k su +im peach +e spionage +! - +di ment +cur rant +bio de +commu ting +by ron +ðŁĴĵ ðŁĴĵ +shad ed +tr uro +cray ons +ar ne +h sc +fre aked +dram ati +fle ek +u cd +marl borough +^ - +cross ings +mal o +black ops +bin ance +cho ked +chen ey +pl o +ge stures +val edic +ryan air +rem ington +v cs +mc kee +ec z +be gs +nail art +mayor of +happy fathersday +war t +pet itions +n ingly +clean energy +bro x +sl alom +exist ent +ab ay +ug liest +tom p +stom a +sel by +goal scorer +ben ji +overwhel mingly +lan s +semiconduc tor +south korea +re scheduled +sk yl +en listed +dow ski +si del +rosen berg +nas ser +white head +pri us +har are +en n +ry der +í Ĥ +mon g +clas ico +transpor ter +po tty +is me +** *** +vic e +sk it +ode ssa +l mp +her n +raci ally +pin oy +paragu ay +obitu ary +go es +bu cha +side walks +angu lar +un constitutional +transiti oning +i bu +gu ys +un packing +oooo oo +black girl +ber gs + ¯ +wordof theday +trump train +thunder bolt +m si +fasci sts +ठ¬ +t sk +collap ses +raje sh +loveis love +migr ating +set back +ðŁĺĬ âĿ¤ï¸ı +t els +safety first +nar rated +jae joong +un answered +lique ur +en nes +dal go +bill ings +salt water +mer maids +lon gs +clap ham +we arec +pic collage +n ach +h ace +pois oned +lo th +ag na +adel rey +guar dia +poli shing +peace keeping +d all +p isa +la pland +process ors +de andre +so bs +p once +dra ins +c be +ðŁİ¥ : +spla sh +meat ball +fon tana +worcester shirehour +ne v +bri sk +b int +ac r +po x +cay enne +skril lex +j fc +hahahaha hahaha +gla s +en gul +tempor al +oni zed +con cre +com pose +vibr ations +plant ers +fer t +criticalrole fanart +t bli +sch allenge +huck abee +munici pal +iam bic +radi os +ne vis +dura bility +mc cla +horse back +inst itutes +ful fill +atta ch +ate ur +ak an +resi sting +illumin ation +hand le +hair care +om ent +macle od +ka iser +g no +bear down +ly f +gl omer +distor tion +z m +san k +roo sters +is now +as ports +ag en +wo ken +st george +ro mper +my le +econom ists +ru to +t will +health and +d ito +ws l +tair p +pra kash +mic heal +h ts +w rights +kat su +fioren tina +defen seman +d itch +var sity +texan scheer +ba ham +sc anned +we il +seduc tive +ðŁijį ðŁı½ +fu e +er win +dav ison +ter ran +moo ds +wool f +re source +@ . +cu sh +ðŁį ° +regre ssion +cur led +la zer +jo anne +ab bott +mo z +down ers +mm mmmm +valent ina +k hair +dream t +cro ok +che k +ste aming +nephe ws +cl eric +as ober +indefin itely +w ye +us news +joy ce +flu shing +wynonna earp +ron do +kis s +hot dog +bar ns +sax ophon +far ley +gas p +decre asing +al way +pe x +l sd +shi ft +p outine +ra zz +rescu ing +ni ko +ho ch +cc l +u aap +n ts +m car +il wx +conqu ering +ket tering +stur dy +delay ing +sto k +vani shed +cath ar +bin gham +in v +ic hiro +he mo +budge ting +[... ] +be ss +sebasti an +slow ed +ðĿ ij +musli m +stun s +acton climate +ve a +se ton +rose tta +oun t +hard in +flu id +ca w +ðŁ¥ Ĥ +yach t +un l +sp hy +provoc ative +or ic +is back +__ _ +nicol as +gy an +loo se +fl in +reb ate +: :: +! "@ +com icon +she ff +down stream +chic hester +beach life +mom life +diabe te +ar ra +van e +ok u +ye o +man go +try out +app ell +he irs +arjun a +dd u +na veen +movi c +soci alists +s back +criteri on +soyu z +k her +da z +yol anda +wine oclock +re ina +one w +leon ard +en dez +u bs +support local +facilit ated +carameli zed +b pa +vuel ta +my tho +m ami +spe are +nbap layoffs +fe vre +nick jonas +im print +c so +craig slist +la salle +gi deon +ha doop +dis regard +w ud +tu c +ma gee +acou stics +ta a +qui e +pol a +cr t +dw yer +dis sec +capit ol +men tion +kn oll +he igh +fin ders +plac ements +l se +indi ra +gur i +madhuri dixit +kingdom s +iambic pent +geor gina +je ky +conflic ting +bay an +aga tha +uph old +dr on +vic ar +ex pat +periph eral +pe ssi +fa f +ance stor +? .. +wid get +pun c +comm enced +beav s +air waves +ad dis +po a +de sses +co den +vu e +ru pee +kar in +spo ck +m sy +ภ° +pr ick +fill more +ti fication +thing sto +sar de +em ile +pere ira +n ad +bright ening +arre sting +wo king +usc g +sp ill +raspberry pi +hu go +ite c +is ma +cuff links +optimi zed +oc c +mi wx +en ka +el ited +afford able +sa kh +coron ado +ho h +at ul +ai oli +jim cantore +accoun ted +vin ay +her mit +groo ves +ran ch +r illa +we tter +ou tof +veter in +ni kov +ki an +fair banks +ram apho +n iti +k ko +ru sty +ne stle +tv xq +shahe er +âĿ¤âĿ¤ âĿ¤âĿ¤ +penn ant +gem stones +dem debate +ðŁIJ Ĭ +auton ews +support indiefilm +mach o +ve x +new sat +ne ti +conce ssions +can died +yof the +mac au +den ds +cricke ters +san iti +mari ano +gh at +ar toftheday +¡ ľ +e gos +gen oa +chat bots +bri er +al labout +mon ty +spi ed +r tr +comfor t +sni ppets +real time +gra in +exam ined +en lightening +tt u +god bless +release the +sing ular +ki ans +ha ka +sor ren +defe ct +mar g +equ ities +d orian +su ka +per l +aishwar ya +pul lover +preci sion +fair way +ne ve +rive ting +vill anova +en com +ak o +passion ately +europale ague +siem pre +x vi +enligh tened +c fr +âĺħâĺħ âĺħâĺħ +wast eland +is f +new comers +emergen cy +amphi theatre +- . +text books +figur ative +tre mb +pe sc +ab hin +ab bot +ac acia +har ds +por sche +kau ai +el isa +car rick +abo u +elli er +be ch +neu tron +galap agos +ru ben +in nis +how to +nun s +sab ine +i ac +clin ched +no tori +fi ves +cairn gor +per i +gr c +ðŁĴ¯ ðŁĴ¯ +mal m +twelf th +di ff +rout ines +marty n +lin den +synthesi zer +nu mber +game cube +fal kirk +byz antine +queu ing +gr ill +scal able +char red +rou ting +her bali +gri zz +ðŁĺŃðŁĺŃ ðŁĺŃ +tol l +termin als +l pc +ab d +war mups +remo vable +¯ \ +vi go +pap aya +ne ve +lov ingly +jo kers +ib les +sse tt +poten ti +pel e +gi gi +sadi q +leg acy +son o +ru pees +retar ded +ele e +par r +fi ance +ey re +say ers +pend ants +mak nae +al bans +adap ting +p ff +pu berty +ji u +ing rad +hypocr ite +diplom ats +phys ical +rob by +bon sai +ãģ · +f att +catal unya +âľ ĸï¸ı +ro ma +more land +so e +conver sions +stl blues +shol m +gra ssy +pra do +on u +assaul ting +> _ +sett es +dis graceful +aph ra +âļ½ï¸ı âļ½ï¸ı +ठª +kil n +goal tender +s ru +philanthro pist +b als +th n +stu den +sando val +dogre scue +eli ons +asse ssed +lar go +hec tares +sh rm +sa if +cle avage +no ches +n ene +fat alities +cur ing +clean ser +al es +p vp +south bank +pizz eria +marsh als +kni fe +an dover +tbli ghtning +sr sly +ou te +digi mon +timesof india +prome the +le bo +f su +wit z +rever e +man as +mam ba +ch ica +gu an +exhibit or +csr racing +d ere +xx xxx +gu sta +story time +ston ey +organ ics +and u +se am +min ogue +anushka sharma +ab a +ðŁİĻ ï¸ı +ugand an +chro matic +as sn +document aries +sh t +ru paul +loy d +k ats +e us +ite ch +me dusa +pan ty +kel logg +et to +talla de +sha a +do st +p ms +mari ana +je ster +croo ks +ðŁĶ ¬ +min danao +ind hoven +ðŁ¤ ª +le xi +tv n +jan is +co te +ãģ Ĩ +ser rano +iw m +ðŁIJ ¬ +k ke +distribu tors +cap u +counterfe it +camp site +ag gie +ðŁĺ ¼ +chhat tisgarh +~ @ +state u +san di +prevent able +cl s +can ne +mm c +i ver +sa haran +pal is +night out +do s +ap ia +absc bn +manag erial +aro se +mo wx +aro sa +ðŁĮ ³ +under dog +remo ver +astronom ers +lent ils +su scep +smoo ther +pend leton +fau cet +e mory +dal mati +af cb +tic us +exem pt +en rol +d heim +ðŁIJ º +restric tion +star fish +sto w +snor kel +thunder birds +she ad +homo sexual +dy n +as li +andre tti +dou che +dom o +tar mac +slu mber +pr onto +first dayof +mini ature +mari achi +argu s +recomm ending +mobi les +in ce +illustri ous +or c +adver ts +gr its +wea sel +pag oda +over pass +gre ys +maxi mus +arma gh +wood land +sun ni +ðŁĴ ī +ë Ŀ +ti one +soci o +ho s +ðŁ¤Ĺ ðŁ¤Ĺ +wind sor +subsequ ent +munch ies +id h +exclu ding +e mi +cu th +z ai +week days +law suits +barn ard +Ø ª +pe tting +net es +mul ligan +pharmac ists +ra quel +e ton +cran ston +gil ded +cle ary +ce ph +ra a +pam per +lombar di +as in +sher ry +pro d +for te +ari anism +buffalob ills +æľ ¬ +ðŁĶ¥ # +uu u +just ices +car ina +nat in +mas low +dro oling +cog nac +cam ber +el ong +r dr +in en +convic tions +am use +tro ck +harm less +visit ation +gen omic +bl and +beno it +chim p +tuscal oosa +gre asy +x po +gil t +se q +per mitted +christma seve +book s +mu e +old school +human right +be ati +ðŁĶ Ŀ +sh at +sculp ting +h wan +fern andes +sci utto +fu entes +endeav ors +maid stone +un paralleled +shou ted +queen of +mer c +band ic +ve da +sel angor +pi le +ja han +intimid ating +disapp ears +cl ich +za ha +w urst +hi v +fod ils +cor dless +aaaa aa +hy dra +bel inda +e els +bu f +su staining +rugby league +no c +brig itte +( ðŁĵ¸: +tromb one +soo the +smo g +ad p +stab le +ing ley +diagno se +ms g +we ss +tic keting +one e +nsw pol +e up +auto psy +adity anath +sun down +river front +si ya +p is +hier archy +dur ango +di jk +ren shaw +he aps +epide mi +david bowie +interne tof +dd i +nation ality +mb ar +air y +win der +w alia +elli ott +c x +bav arian +pl att +an tw +wi wx +sof ter +ne ha +h eller +th and +dani ela +bo ast +degra dation +ðŁĴ¦ ðŁĴ¦ +transform ing +man e +av ut +ðŁĺĪ ðŁĺĪ +vo ter +the e +t ate +pu ff +in door +sop roud +boy ce +boris johnson +wait in +immun ology +ðŁıĨðŁıĨ ðŁıĨ +âĿ Į +street food +liz asober +cavali er +c elia +need le +motor ing +g ato +, ) +ra de +harve st +t ms +jar pad +on ey +air men +v re +impair ment +abhi shek +snoo p +l ant +fam ously +bl ou +s ze +g ander +un touch +tu f +dee jay +col lateral +b ind +ðŁļ © +pin ning +ic n +' ; +the economist +ul tram +worldwater day +ti poff +the i +feed ers +campa ign +sc umb +day weekend +yo m +pe dic +h ough +ps v +pl in +on de +boston marathon +az zy +* _* +con ley +thi ago +hoo o +gal erie +luci d +je tt +gl itz +final fantasy +achiev ers +y ung +peregr ine +op hi +dam es +biom ar +âĺĢï¸ı âĺĢï¸ı +sk c +l ics +fl ank +ar rahman +ho of +uphol stery +t ats +wo z + ¿ +snor ing +ra er +l ju +ap d +pl ating +kan u +im ation +fragr ances +m ra +mor ay +mo tt +im muni +hearti es +bho pal +tim ers +g ata +color way +car nation +win get +si ghs +s ville +optimi st +chate au +olympi ans +ci o +singer songwriter +ny o +fi bers +bur ch +ag ro +mil ne +ig bo +cr amer +ation als +dan ube +pad ma +nor mani +en forced +bre ck +boeh ner +ar den +sur rendered +pros thetic +om a +ha iled +calcul ations +w fa +bi b +fcb live +fon da +west coast +que sts +friend ly +to wie +fit ch +bal ot +star dom +scrat ching +ho sa +thi ka +o ven +stro ke +out post +pharmaceu ticals +hi kari +mu y +af d +fallon tonight +squ at +or u +dra ined +chocol at +ë¯ ¼ +wor ths +ri b +mu j +that s +residen te +it el +boo st +mi gos +mul led +la a +etsy shop +don keys +me k +p tc +flin ders +e hs +ro hit +mu ir +g ad +compos itions +åĨ Ļ +combu stion +i kh +yemen i +wav ed +gar ci +ak os +oo ds +fu sion +se que +s lan +pl ur +kic chasu +shenan do +s ams +worl den +horo witz +with me +mic robes +k ki +ðŁĴĶ ðŁĴĶ +w su +patch work +fre er +y aki +the art +symboli sm +mil er +bt n +ma bu +side kick +motiv ates +sag itt +natur als +serv iced +ps ori +pa ola +qu ig +i badan +gi ggs +ë ³ +sciento logy +si oux +salam at +d res +cad bury +d hawan +ci ón +_ ' +swa pping +maris ka +james bond +explo sives +ay les +af er +s agu +cen sor +tom a +jeff erson +ring ed +par tist +ir responsible +aguil ar +vac ay +equ itable +altrin cham +ac ur +man ish +ger min +schoo led +pu tter +ed ad +nav al +toast y +sol areclipse +dish u +coy ne +ac co +mu ck +mar an +el os +len der +cro ix +worth less +ha ber +gun men +ðŁį ĵ +zen ith +t enders +hur st +hol tz +itali ans +car low +u cd +characteri stic +bun g +av l +u th +sa sia +rs l +red man +neighbor ing +green peace +sti ps +follow party +y gk +en os +omni bus +na issance +chri ssy +secu re +call back +ji hoon +memor y +block er +l anta +daf fodils +bil t +ffer ty +fau st +ie c +nipp les +so g +m nd +jagu ar +bol dly +ab poli +pro position +gun sense +evan sville +cu tters +we go +dou n +do x +stal lions +ka j +shi ppers +j awa +vol o +le ven +pap rika +kov ich +jor di +induc tees +app alling +dial ysis +allevi ate +âĢĶ âĢĶ +pie ter +mid wi +q tr +juli ette +inter mission +haw ks +act ment +one ill +k lin +vam ps +fam ous +cou ld +autom obi +da an +west end +elli p +nh c +mel anch +web series +ton gue +snat ched +smy th +tan gible +sl i +e asing +bar stool +over lay +afford ability +ting ed +ter as +ay ush +wanna one +rh ine +dan a +sh ana +kend al +fer tile +w ir +repl eni +lar vae +is ro +con vos +ab brevi +u cc +hun gry +bur rows +ag er +nav i +mat in +du per +cer n +ma don +ķ ï¸ı +é ģ +tu ps +hy att +sh ep +friday night +wis er +hei di +hat ton +p gh +foun tain +wrist bands +ahmadi yya +aeri al +subscri bed +so los +m ace +sla yed +for fe +dul ce +christ mass +arun jaitley +viol ate +ob stru +ni eces +w vu +idy l +fa ze +pre serves +infr inge +premi ers +inter vals +agen cy +( © +stand alone +di mes +bo er +param eters +ge tit +ðŁĺĺðŁĺĺ ðŁĺĺðŁĺĺ +tu lane +for given +scol l +mb ps +smash bros +rob bi +prima vera +ali st +ghost ly +ay at +ye ats +impre ssionist +ear phones +caul field +wai kiki +sal ute +sc ou +mu ay +louis vuitton +bak hta +ado g +inven tions +hur d +forec lo +stream line +thalai var +ch snews +will ard +t sn +euro parl +cru sher +my sore +gro wer +ra ping +pat ti +g den +sm w +muf ti +kid man +ab r +soun ders +skep tical +ðŁĶ İ +sun dar +i me +fer g +feather weight +ar lington +pas qu +ag azine +wearab le +nati c +mccl ure +inter mitt +hor de +six ties +car te +bha v +ze al +experi ential +ador ned +som mer +eno te +hypo thesis +stin ky +pro to +dead lines +vo gel +mus ings +monc ton +gu ter +f le +aci on +voice of +ta sha +inhabit ants +type face +s ba +bts x +ðŁĶ Ĵ +wor x +u hc +jo ko +cell ars +gor o +continu um +... & +weather cee +ha p +sr k +ris ers +lonely planet +un named +co eur +ðŁį Į +the world +ili ke +fa sten +ami go +ri ba +ramapho sa +staf fers +had ley +? ?" +fi ore +sal ut +hu ff +bez os +Ñ ĭ +ra der +kam ala +in line +fill ers +um atic +all in +shat ter +re in +o ku +ch ases +fla gged +baby metal +water stones +ts b +cut out +op hel +aam a +rockab illy +sto lic +jet blue +ich ick +down ton +uzbe kistan +pat na +la q +gr ange +) _/ +subsi di +sc p +newsc ast +it sa +twee tyour +e mor +archae ologists +uni fication +por ta +q x +protec tors +pro hib +charis ma +car tag +ren fre +scul pt +guwa hati +de ma +boo p +unf pa +dex ter +lay la +alleg es +sou ps +never again +l ys +cal c +bar oness +visu alize +ger ber +absor bed +i ers +a han +fon tein +detec tors +verst appen +sv c +formul ated +ac dc +li x +in competent +bh k +lour des +water house +snow ed +appreci ative +sig ma +lizasober ano +pen ned +pay check +tall inn +fanc afe +par isi +av alley +vi g +ru fc +hard ship +so cute +po ise +ì ¹ +roth schild +k ly +???? ???? +l hp +il ay +f hs +am ad +ide als +brad bury +bal boa +nic ot +kid nap +wol ve +tas manian +op t +matthi as +ãĥ³ ãĤ +super markets +mylittle pony +me lee +li ster +gr oun +fe dora +kind ness +en en +bra hms +¯\ _( +ros well +mar lene +ic u +re formation +or ail +he brides +dispar ities +terrac otta +swal lows +re id +influ encing +flu or +den e +tum our +blon des +thunder bird +sh eva +moga dishu +ka b +cre eps +i ving +ene ed +anno y +âĶ Ģ +intri gue +enqu iry +ar aj +tur al +kuber netes +end lessly +divi dends +tor a +ti sh +commemor ates +un ra +tri b +pon ty +ne m +diss ent +brew ingco +ðŁĺ ½ +nor mali +bi of +( ... +chil len +ì£ ¼ +mell on +av is +mccor mack +ing ra +enrich ed +custome rexperience +testo sterone +snu g +sett i +ger onimo +inqui rer +bre aches +very thing +bloom ing +mu ra +dispo s +bi de +de va +shade sof +in trin +sh ev +s ven +nayanth ara +gan esha +c ws +ber ta +label led +use um +nick named +ma han +car uso +ap ur +ðŁij Ĩ +w q +orphan age +discar ded +mag nu +lu e +je on +bridge port +pac ing +mercur y +( ðŁĵ¸ +marx ist +amphi bious +transplant ation +stit ching +then burg +gradu al +ãĤ Į +ro ft +ma ils +ine c +guy ana +dopp elg +ver o +re write +head less +harb augh +gate way +car sforsale +sw i +st is +mach t +un de +sura baya +stap leton +nur turing +mil ner +ya o +lma oooo +ko sh +arsen al +k ame +er ry +ar royo +dis misses +ru bbed +rc b +lew d +dil u +and or +vi de +ur in +inter sec +ha ar +al b +year swith +app leton +é al +ul livan +suc cu +monter rey +d mx +artem is +ron nie +farm land +s football +gro tto +anth i +ãĢ ģ +à® Ł +vid ya +jimmy fallon +ൠį +t zer +gravit ational +w thr +u hhh +e hr +tin ker +ti juana +scran ton +ram charan +bar clay +re van +m si +ka p +wr s +we thenorth +tor al +sat u +gro m +fac ep +erick son +z yn +se dge +oo dle +spur sofficial +ds p +sic ilian +soli hull +recei vers +ladak h +hend rick +ther i +presi ding +mc guinness +litt ers +gun nar +gh oul +wi b +n tv +kar o +fro ck +b lau +ampli fy +all is +ul lah +memo irs +kh loe +intercep tions +pet day +lo oney +con fin +ch ay +piyush goyal +frequ encies +ut z +event ual +warm ly +obli vion +an ka +ta it +âĿ¤ï¸ı . +director ial +ru lers +prince s +mu ck +stur ridge +deu ce +abri dged +bagu ette +un cles +pen du +min ding +forre ster +av ila +wall er +wall street +ment or +hin o +high way +crom well +fanart friday +mb i +co yle +a hi +tro ve +spie gel +pay tm +mcin tosh +jan sen +nit i +nash ville +len o +leicester shire +le gos +dic t +ðŁĵ ½ +sp ad +beverly hills +sy rah +separ ates +z ain +un fit +dra gs +tan ia +over flowing +hri thik +haw thorn +z ani +mac far +fi de +to tem +pe ds +fundament ally +cal ico +sin ner +j ä +hil de +ds d +ten ay +ta hit +mil f +lie b +inform ing +up lift +ra el +mortg ages +lec t +ii ii +guillau me +compos ites +old smobile +l end +gar th +com mish +bapti zed +scorpi ons +ru cker +bringback our +alli ance +thalap athy +tal i +sp ans +eri dge +wither spoon +lin da +sky lar +kor n +hom s +Ä į +sil enced +caf fe +ar ty +dist inguish +to wed +pun g +jessic a +ear nest +beau fort +t ama +study abroad +si khs +new bie +nav ratri +mar ble +loun ging +lit ter +dal it +so sa +iz es +gra de +com promising +tr iton +de tta +v j +chau ffe +spec tral +powe red +montess ori +artic ulate +hal ton +al co +ye y +mn twins +acoun ty +ðŁijı ðŁı¾ +âī Ī +mad men +kal a +gru m +chi k +ati s +su me +akh tar +job search +high lighter +bo ath +âĦ ¹ +tar zan +lam bo +âĽĦ ï¸ı +ox fam +dump ster +pretz els +mac os +incl ined +fac tual +adverti sers +shu i +pu ree +ml pfi +anti dote +cap o +pa str +merc ado +but ton +ar min +ag g +lol la +horri bly +er rands +christop he +time snow +monday motiv +li ss +scand als +mc i +dispropor tion +âĺ İ +sur pass +samar itan +so tho +pu rest +fl att +trivi atuesday +delec table +leop old +hermi one +chou dhary +en rich +¡ ¡ +subsi diary +ine qualities +bachel or +auto immune +la kota +i hop +ad jec +the simpsons +sh es +se k +gret chen +up stream +hin akhan +coper nic +x tina +lu g +tough ness +e ad +cli pped +bi us +sl v +fah ren +dee pak +ca u +x an +im mature +dig ni +bo bs +shred ding +but tery +accommod ations +de ven +chun ks +super league +sky bet +kil dare +je et +ë į +ce k +wrec ks +pro pane +oh l +tb d +quo i +trum pp +mi mo +reluct ant +ver ne +o ic +ma gh +ar nau +se ver +li dge +stair way +kicchasu deep +ðŁĶ º +mach ining +aama admi +ot i +c da +al it +pan y +inst alls +ac ct +e shop +di em +hard well +fulfill ment +sc afe +qu ack +extrac ts +swee tened +fi ghton +f di +d inger +wal tham +us ur +refe rees +seok jin +gran n +af rin +th n +sch af +par cels +bet is +amar ine +nom an +kh tar +mor itz +cou pling +bar ons +ðŁIJ ¸ +à ¸ +sl p +sad ler +x ander +tri ad +mc millan +kh z +divi ding +ìĹijìĨ Į +dar yl +zed d +le ys +pla ques +flu ori +tipper ary +on nell +di dier +lang ford +im c +the sun +bir dies +ar cha +ye ssss +t di +dar ia +cand ace +al tam +pal aces +ch it +sant am +event ful +book of +ad b +mon stax +cre ole +co el +âĸ ½ +we aren +sten nis +she ath +ati sm +gron ingen +mlpfi m +le pre +wrong ly +rsp ca +rendez vous +acknowle dging +pel vic +solic itor +sla ys +nue stra +lo d +is lander +fer oci +fashion show +ra ss +dge on +adole scents +sma shes +negli gence +grate ful +ved ere +sw oop +ing l +apol ice +vand alism +gan n +jo ao +di supdates +zimbab we +under age +radi ance +w of +bour geo +pla s +cr ani +gh ue +wrec kem +warran ts +re form +jim mie +at wood +ys l +neil himself +l bj +i man +tan to +nois se +ver bs +equip o +al together +mam ent +l ice +dou glass +tier ney +pri med +j hal +furn itu +braz ili +v ill +past els +n ison +u ff +paral ysis +jay e +im po +ðŁij ģ +strate gically +pakistan is +was sup +super bike +thank u +tru elove +sha ikh +israel is +vi p +to g +li en +la ker +grey hounds +cul ars +bian chi +balot elli +ar ran +loo s +str ates +he bron +ar vo +sunder land +the al +tomb stone +sand man +c pac +thanks giving +love him +lat ino +an in +aka if +ĭ ãĤ +tor quay +di est +alli anz +ðŁĺ ķ +golf club +cl lr +wal cott +sch nau +promp ted +nomin ating +len nox +val et +mon ro +may ward +e ph +ðŁĶ Ķ +inter oper +r da +re flex +arm chair +ê° ķ +stri pper +por ti +ph arm +ham za +ni reland +ne ue +h pv +port foli +sun burn +fris bee +be al +bapti ste +x h +ty m +pr ati +o vers +haz rat +deser t +der ry +us ky +em mett +ach arya +)_/ ¯ +shu d +may a +ham ill +ra im +nr c +fitt ings +cur vy +ðŁı ĩ +ster ling +ॠĢ +wal kin +short cuts +mil ly +ast ur +alpha be +pl i +pe z +miss you +rad ford +ml g +ta eyang +notjust lakes +du mps +seren dip +le ur +ra ving +e ster +de priv +absc bn +ðŁijĩ ðŁı» +scar city +o cr +mean ings +cap t +da hl +fer mentation +bri oche +to win +out lander +massi mo +en cro +ðŁ¥ ³ +buil t +po tam +kir i +tm w +monit ored +k ites +peoples vote +gray son +íģ ¬ +afri ka +a dies +i vote +gy ne +g annon +di x +c mc +ou ral +fox andfriends +bel i +ig ne +gl an +katrin akaif +co politics +qual itative +p si +lu cci +disc oura +âĺ ® +kel li +gau tam +carac as +reale st +pu la +in us +hill top +make aw +atten borough +tw y +r arity +peck ham +ma hon +corn elius +clin icians +ton line +tb i +paradi se +ka si +inev it +fresh ness +colling wood +lun atic +defen se +cop d +in fra +wain wright +sains bury +alab am +te ma +lac o +chec ker +releg ated +tren t +stal ks +huff post +bhubanes war +ast ral +share your +prim rose +hi me +cat an +end ment +en dow +cle mens +mal oney +hil ary +game time +den ise +collabor ators +b wo +radic als +gue tta +ici on +au a +snap matic +sat chel +excav ation +base man +s ão +gn ation +fel d +surve y +shah zad +ma st +anirud hofficial +tru cker +ot ago +geo graph +ethe l +âļ¡ï¸ı âļ¡ï¸ı +s ver +mu tt +internetof things +ancho red +wh ouse +bang la +bal main +ç¹ ĭãģ +break fa +á Ģ +twi ster +te tris +ca v +stag s +g z +au b +stor med +hel ens +yar mouth +st asy +gustav o +co sc +vin son +up p +sc ricket +assump tions +app e +nu h +u er +pre mise +n aga +e amon +coron ary +na f +north side +el mer +ro tar +out lining +el f +re surg +kat elyn +in can +hyster ia +ce e +am bani +pro lly +Į ãĤĬãģ +ax es +san jose +rem brandt +mag pie +even ly +scor sese +qu aint +f g +b buk +indian football +weare all +spd wy +pis ces +ec g +âĺħâĺħâĺħâĺħ âĺħ +pre orders +: | +ni pple +sal azar +ju me +jail break +min n +bas sett +ze tta +jef free +ad jun +tic on +san diego +drink local +chol era +solic itors +o bo +com post +ni an +wr a +tre ach +ic ic +profession al +del ve +leg ate +histor ia +cro issant +con noisse +nam o +palli ative +chem trails +i ority +global warming +comic art +behavi oural +re sted +li as +cli mates +Ł ãģĦ +rut land +nou rish +menopau se +hot ties +demen ti +ve spa +mel ville +anal ogue +tz man +str ung +im perfect +gl are +cir cling +ros berg +rec o +oc ity +lo ire +em be +do ssier +ne el +nan do +me a +gal vani +fin esse +ag p +berke ley +asi m +âĺº âĺº +quil ted +ish ere +un matched +po tion +for z +at re +selfi es +juli ana +ðŁļ ¶ +âĸ º +mel ton +âłĢâłĢâłĢâłĢ âłĢâłĢâłĢâłĢ +spin rilla +pur cell +ed p +at leti +tony awards +ra ja +pro gno +mol ten +stu ff +p ally +nobel prize +âĻ» ï¸ı +spiritu al +spe ake +sa sha +bri um +tru ss +critici ze +assassinscre ed +yor uba +u lo +fire man +workin progress +ef cc +fla res +ro bot +hi kers +cl l +shado wing +pat sy +leh man +c ns +å ± +guad al +à± į +ra pe +r honda +paralle ls +son ja +langu age +land ings +z ola +cr amps +bur ning +apprais al +jol la +ham m +kas a +gul ly +f go +uly sses +ri be +ðŁĴ Ħ +ib u +eti enne +bri ar +fin ely +comb ating +y ql +go tham +we chat +to paz +primar ies +l se +iz z +hel e +dispon ible +cy stic +bel ichick +th rush +kansas city +ge om +soli di +red bubble +by stand +cambridge shire +par fait +ast le +ow o +ind ore +stom ping +sm elly +ðŁ¤ ĸ +locom o +adm itting +hol me +clock wise +min sk +mc co +for get +ev p +cam ra +ab ella +yo tes +universit yof +mé xico +silver ado +ric ket +crom bie +pu j +eradic ate +deli ght +y go +glam ping +vic a +du ggan +coun ters +cf d +sc our +react js +pu ram +paras ites +in ki +vill en +stel la +li mbo +ang as +k cr +ðŁĴļðŁĴļ ðŁĴļ +vap ori +mum ford +oli gar +à ¼ +al oo +boo ties +ad r +k elli +dru mmers +av ici +nature uk +ron al +in trac +un splash +le che +g oma +el ine +envir o +bi onic +bu eno +mi k +av in +star ling +em powers +cake day +boy cot +ðŁĴļ ðŁĴļ +ðŁĮ¸ ðŁĮ¸ +v ach +m ci +fractu res +ger i +sk ing +exclu ded +lu ce +ja ve +ig gy +evi den +aki stan +a wn +mor als +luci fer +ha ban +tumb ling +sunday motivation +mo sley +captain america +sch icago +the one +mo td +d ts +ðŁIJ ¼ +rep ell +ii i +locu st +geo spatial +mer sey +immer se +desc end +ber nade +j s +boat sales +win der +cran k +sing leton +candid acy +ben a +ðŁı» âĢį +high lander +ol t +k prs +healthy lifestyle +four teen +end the +ith aca +circul ated +r ans +pre valent +ha vas +splend or +roo ster +kalamaz oo +jewell ers +enne dy +rou sey +es y +cann ons +ornam ental +// // +ren don +win ne +mol ding +eid mubarak +coun tess +simon a +ha wa +fo es +du ster +sb u +por tray +mar ries +goo dday +cho co +achi ever +ðŁĺ¹ ðŁĺ¹ +pre neur +tr amp +tom i +n bat +garden chat +farra khan +ever glades +ab ru +sou sa +se ce +homes wee +terre strial +bar it +sri devi +ol u +mel inda +f rick +can dies +ðŁĺŃ ðŁĴķ +qu reshi +family fun +exor cist +cardin al +ny t +dies el +cu mulus +capric orn +si ology +lor na +dou gie +an die +super sport +c fl +п ÑĢи +say ang +pe ek +ภĬ +lo be +j em +ing lis +gg led +c sn +amne sty +chu ps +ba es +sau er +ðŁı IJ +mongo lian +en et +back street +dr illed +acce ssing +ce o +b se +ai ken +pur r +wor sen +whe res +war k +testi fying +bu ri +bla st +aw g +ðŁĵ ĭ +re defining +hear ing +u ci +c mp +bon i +tail oring +ta ji +noc chi +em t +stephen king +ne et +compla ins +campaig ner +luci ano +twili ght +ti esto +pas sports +flo yd +cathe dr +na ked +caregi ver +b coz +ade cides +ku ri +ly k +br aries +dren ched +disc lose +ðŁĴª ðŁı½ +le blanc +je tty +gar ty +chip mun +b su +rhyth mic +ic z +fri d +anne x +ame x +solo ist +lanc ers +arro whead +speci fication +simul ated +na is +inver te +bo wing +wor ship +f z +abo ss +sha q +ì¶ ķ +challeng ers +an arch +aamaadmi party +ãħĭãħĭ ãħĭ +suffol k +so corro +sn ell +cla dding +absor bing +shaw a +particip ates +ðŁį Ķ +book stores +bak u +seap ort +ko jima +gab y +pack ard +electr ician +let it +mo wing +fa wad +young jae +hot mail +men ing +u rie +intim acy +con ti +: ") +lifeis good +in ciner +i dri +craz iness +jour nos +fran chi +bott len +al da +ff es +k x +south we +air a +clay ton +sco ti +f j +bri ga +ðŁ¤ĺ ðŁı» +demonstr ators +y z +stor k +na q +casc ades +travel chat +plat a +pad ma +fran ci +at tain +bat girl +lom bard +hoo s +d dos +neon atal +discla imer +r ss +r ant +di sen +tex aste +so cal +frac tal +cam ry +stri fe +sn acking +mu h +sant ander +mor ons +gra f +par ades +hu ston +dru pal +mi ento +kir stel +hy de +vom it +forti fied +sphin x +da v +bir yani +win nings +s baseball +mer ged +lovel ondon +ling ering +dream big +car leton +liveli hood +djan go +astri d +gri ds +down e +bru ised +s ne +scarec row +hel ium +f nc +bi ggs +an ter +restor ative +em pires +ab del +life style +kiwan is +colloqui um +me en +pr ick +anti que +ze b +mi mic +edmon ds +ðŁijĬ ðŁijĬ +q ing +pp el +mc gill +interpre ting +âŀ ķ +rash ad +do ka +narr ator +electro magnetic +ash by +sau ra +iran deal +âģ īï¸ı +krish nan +in di +ff en +bre a +os man +multin ational +chi ppe +recruit ers +aus biz +p ounding +re gen +cur sor +refu sal +mac s +in ak +ax ial +wa ifu +up cycled +hindu stan +cas sini +carly le +scrat ches +re ef +man atee +eat ery +ðŁĵ ¢ +un condition +sen pai +on ther +comic book +pro sciutto +de mar +mi se +ma ge +fre ec +aye sha +al der +android games +ley ton +ho ck +door way +chicagof ire +aali yah +sw elling +bi x +. ðŁĺĤ +evan kirstel +torpe do +kon stant +genevie ve +ma ia +ha user +do torg +hide ous +fi k +sp raw +e ek +z appa +wan dered +' ' +ra jan +bam bi +( $) +wid ening +tool box +sa ir +illumin ating +pra ys +out patient +i w +day o +lo b +sw fl +sha des +gu ms +coo kin +ko di +gri ffin +traum ati +ste a +slaugh tered +god bless +air time +pseu do +b sa +hau led +ar if +à¸Ńภĩ +le l +wc po +mil iti +char ters +worl da +ru k +k gs +digital india +is able +idyl lic +esp ino +marie tta +e bo +team canada +ab our +wil ton +rock stars +fav ored +phys ic +wrink le +tb r +d print +ball arat +ad al +z ey +ðŁĺį ðŁĶ¥ +tom lin +mt r +pal sy +fener bah +tight en +phil ia +ir oning +ry u +b ant +enqu ire +ca ir +abur ger +tru n +green berg +chau han +ir ina +sh ani +trend setter +pre tt +zaf ar +alo ve +v ici +pan ic +no o +lu stre +disrup ted +bal lis +son sof +mon si +inst ac +ake st +ëĭ ¤ +kw ame +horror movies +distric t +sau cy +mb an +ar mies +with drawn +med ics +loft us +er oom +be kind +ar ns +all on +un ison +davi ds +cr at +nicot ine +so or +sm x +on co +cospla ying +zombi es +har ms +e ger +ro sy +moon shine +fe in +ce tt +du brov +reg ents +ben itez +ðŁijıðŁı¼ ðŁijıðŁı¼ +ste c +m alia +prioriti ze +ic eland +ft se +v amo +lam ont +homo sexuality +bre es +regu i +cb p +te j +sky sports +deter gent +sha sta +de rel +conserv ancy +colori zed +accol ades +vis o +show your +nan ow +bice ps +us ability +bi m +dailys ketch +pearl jam +stran gest +mega deth +broad casts +bar ren +ar ton +chri ss +confi gu +lu res +is the +e ul +railway ana +global health +gi anni +u aap +s lum +consci ously +ab re +n up +bud get +v ada +e sch +real ness +er ased +th unt +be z +armist ice +ðŁij ¹ +sh run +o led +driver less +ðŁ¤· ðŁı»âĢįâĻĢï¸ı +won dr +sk an +sal aam +mother land +h wang +gen o +gang nam +tw right +endor sing +en ic +ador ation +pau sed +patric ks +do cked +plat te +ff xv +ethnic ity +auto show +side show +after life +re located +orphan ed +food network +dare to +and ra +sla ps +v live +swim s +re imagined +mist le +re vise +real ity +bhar ti +ðŁĴĻ ðŁĴĽ +late st +prou dest +gra sses +lan yard +fresh est +carcin oma +anom aly +zieg ler +sum ner +ly rix +gor g +is d +av el +swild life +me squ +john cena +euro league +sab er +master ful +yar ra +cogn ition +jacob son +abo lic +sir loin +shuk la +moj ito +su pere +st weet +me z +e sa +rudol f +gur a +where you +tt m +win s +trust worthy +ny k +bra den +table top +good food +es on +be k +lingui stic +gra ys +ch ath +h cs +mon i +de ans +cu ssions +ch ell +slo ws +he mi +d app +shar pie +boo sters +a os +str ack +se dona +mu eller +hard wick +or nate +thor a +sal ud +o twol +ch um +mi ho +for age +thel ittle +tear ful +ones elf +min dy +sm g +gmb h +emer ald +ðŁĶ´ âļªï¸ı +tu tti +recep tions +re vising +i brox +tope ka +sal ami +expan se +i books +dob son +cli o +at s +ðŁļ Į +mo ha +is ance +shu tters +moo t +jan ine +marvel comics +jor dani +pos er +kenne th +hy ung +de ja +ase ball +speci ality +eu ston +classic car +had ith +ðŁIJ ī +chas ing +iz o +gros ven +ag lia +thisdayin history +t row +om ile +hu ar +by n +sal ine +div ine +demon ic +ty ran +han dover +revit alization +pa ella +cryp tic +se dg +m end +dun kirk +bre d +wal d +sport scar +a ard +whe aton +da ener +k lan +br t +bakhta war +spi res +schu bert +ro ti +poli sh +o se +ag ame +wonder con +prote stant +bo sa +ðŁĺ Ł +d ü +joy ride +ger trude +âĿ Ŀ +gil a +v h +tw a +tra v +swal lowed +star ve +la in +ent ren +rei ki +su kh +cra ic +az u +web page +kee fe +hypo the +hir sch +hel le +camp ground +w amy +tra vi +sha hi +san deep +ru i +han uman +dw p +reposit ory +no or +no ff +un real +p ell +black history +har vick +ma scar +pay ee +pa sha +gastron omy +d ÃŃ +ai g +rosen thal +open day +embelli shed +t tip +sun bathing +go pack +end ome +ï¸ı # +invali d +final four +st fu +squish y +ra sta +mo sch +jam esc +die trich +sel a +mel b +el vi +t dp +sun i +sli t +j ha +bi za +spi ked +l li +l illard +vam pi +syno psis +az har +kendrick lamar +ĮãĤĬãģ ŁãģĦ +heart less +country file +air play +arrog ance +pre e +virtu oso +ãħłãħł ãħłãħł +raj u +le bu +for ward +tu g +dro s +mondaymotiv aton +concep cion +thel o +pad i +looo ol +ÑĢ од +it ss +eth ical +end uro +__ : +expend iture +mon ste +mas king +terri ers +ib is +e mber +cu mple +punctu ation +pi per +ir vin +ade e +yy yyyy +flash backs +cel sius +don nie +bo gota +ben evol +the script +shil pa +pro se +fin dia +ze ke +ne ko +do ves +blues lyrix +fro sh +sowe to +mp lo +al ai +sab i +raq qa +wf tv +stro ller +ian somerhalder +ðŁĶ ª +an on +mo seley +! ?!? +sta king +mol y +car tri +c sg +ast or +transc end +ma er +de ux +cow girl +sas k +pun ter +ma ken +o ates +love tt +grow ler +sag in +v n +ssi ble +officeof rg +y mc +sab ar +faul ty +ap ha +ak on +ðŁij « +snow don +ae w +raise the +ðĿ ĵ +grue some +clement ine +sp ing +lat a +worlden viron +mi mic +can aria +bakhtawar bz +ao a +fal a +ãĤ Ń +avi va +you uuu +thi gh +la dders +gu mbo +tz ky +fu zz +plastic pollution +est ate +strength ened +k ant +dr in +cal vert +transform ational +frigh tened +mac lean +elited angerous +ear thy +t son +to da +j nu +.. , +mic hal +i ban +je ong +is real +sim coe +exclu sives +blue bells +ben e +te u +pil sner +pens ke +athe ists +m pu +cartag ena +ðŁĴĹ ðŁĴĹ +million aires +kk kk +it ar +subscri ptions +remo te +ma fi +hin ton +w cc +ho k +ds b +ab leton +sevent y +pun ks +e indhoven +sh one +mcfar lane +lim popo +empha si +à ¼ +sin fo +pe tre +man grove +ch ino +ber tie +play lists +push awards +p af +deb bie +c do +r ino +ðŁı¾ âĢįâĻĤï¸ı +fol ke +bon nar +th ine +sl an +hal ter +evi e +aw some +vul tures +spar ky +seiz ures +âľ Ķ +ram one +ine ffe +al n +pro ctor +ast ra +the voice +gro te +sci on +dead line +am aya +tain ted +patter ned +exce eding +cross fit +kay lee +drop box +ru shes +tack led +mo by +retro gamer +n cbd +benef itting +shay kh +guild hall +gen try +dream cast +dread ed +bun dled +th aw +revol ving +n pt +kylie jenner +imagin ative +ron i +over came +family time +ds burg +car naval +relation ship +recogni zable +cor oner +ho le +fan fic +emir ates +bur ritos +analy se +thin ner +ne es +galli poli +bl r +cat woman +-- >> +au lt +ada ily +nau ghty +ili o +solit aire +mtv br +jocel yn +arun ach +rep ent +south gate +hy acin +essenti al +fent on +and um +it or +go pal +sl inger +po sei +aw il +wi elding +ra ila +eli as +a sto +à ¤ +tend ency +str ata +ker t +< - +im acele +da es +sti mulus +han ley +fit nes +ec stasy +lim ous +ha iling +ðŁ¤ Ń +chis wick +tar ies +sla v +pul i +moderni zation +black mail +b ingham +h fx ++ + +ðŁĩ®ðŁĩ ³ +ni v +we a +profess or +k off +bol ster +su ave +sequ ences +pepper oni +not te +dre n +ãģ¨ ç¹ĭãģ +hs v +o ga +ap tly +z ad +excel si +rin ka +mol dova +min n +ma bel +conferen cing +bas ing +of er +ob si +hamill himself +care less +brief ed +inhe rent +par ish +dub nation +town sville +sar awak +gee ky +doncaster isgreat +was abi +gu p +phen o +dra inthe +carrie underwood +ble eds +bbc world +ane w +alta f +dul wich +ani ston +w ti +sumat ra +gra fton +bl n +me ster +bode ga +re go +es q +an jo +sump tuous +mai sie +ï¿ ½ +wil t +jak ob +el vis +se pul +mu ster +air pollution +president e +happy monday +exten sively +fl ondon +t ls +play ing +pe ed +din ho +var dy +pi ka +n iro +au cus +ðŁį ¦ +nu ll +el ondon +juvent us +imag ines +dis ab +lit o +d ura +work places +promo te +mc caf +wood work +waw x +à® ª +tt ino +shar i +sem per +better together +ðŁijĬ ðŁı» +ze bra +pon dering +en chil +ho m +cosm ic +tan z +mo cked +ec cc +ath ed +abo lish +prop eller +paris agreement +assemb lies +indu stry +fraudul ent +pe sa +chang min +ax x +ðŁĴ µ +irr ational +cu sa +ramad han +octa via +on elove +jac ki +bar ak +taxi der +seri ous +nathan fillion +mc en +ch k +po part +grav ity +copp ola +reading fc +illu sions +j ig +ww x +re sh +ex porting +buzz ard +âĻ ¤ +p cm +lan apar +ko s +arom as +antal ya +ww dc +ven a +phil a +ball in +ðŁij Ħ +quin ta +ma o +f ery +eigh ty +sentim ents +safe guarding +r wa +pu ffs +luc ille +de cath +sl u +nu gent +de ter +braz il +ze iss +super bowl +subsi dy +alter n +hi dalgo +enz ymes +ä ½ +tag ne +hair dresser +adri en +walk out +oppo ses +can tina +bed side +af an +ðŁĶ Ĺ +prophe tic +dan es +un successful +super charged +pk k +exem ption +hart le +secu lar +cli pping +br s +united way +c net +pat chy +ha gan +e en +âļ ľ +var a +sym pathi +never trump +affir mation +om f +ny cfc +ma ja +sur ro +keer th +up scale +sandal wood +mon archy +kno bs +å ĭ +po tholes +hunger games +ter races +na sir +coun sell +welcome to +wa q +se aman +m ita +stun ningly +on theroad +in ability +) !! +bon go +ant v +sp ut +worldenviron mentday +resu sc +y td +fi m +eun hyuk +sa chin +rose anne +cler mont +ape c +am ina +v ening +n antes +al most +sin us +ex as +ty l +ti en +ple ad +lanc s +bur naby +re k +jo om +observ ers +disco graphy +cl g +âĻ ¦ +sn ack +r ti +o ily +crystal li +bru te +web development +topp ings +la f +an is +ad der +reli ving +car lin +battle of +we g +syri an +pon t +n dc +lagh ate +yu ma +sp p +p iti +ro bbing +mart ing +rey kja +raj put +nc ds +kie wicz +âĢ¢ âĢ¢ +vam pire +substan tially +opio ids +nepal i +k line +ar oo +under stand +lit t +u it +thro mbo +sar ies +qu ot +b alling +t tr +s gh +philip p +br ant +ac l +m ello +whit taker +. ; +defi ant +b gc +repl ying +mir ren +metamor pho +sch wab +bul ge +utili zed +pick ering +par don +d sa +ภĪ +doo ley +cumul ative +Ð » +ur gency +e mir ++ /- +¦ Ī +ot as +âı ³ +station ed +grape vine +ar ac +karan johar +f ancy +sau l +coo gs +lgbt q +ا٠ħ +jav i +u mmer +pl l +den is +dai pur +pu ffin +lewi sham +fand om +co pe +ves matter +s ve +hel pless +deo dor +ostr ich +kaz an +friday the +con dor +v x +sophom ores +rob les +cu tt +cli mbers +ë¦ ¬ +sle g +sn f +mac ys +hydr ating +grou pe +po yn +mou lin +hg tv +lmfa ooo +sulph ur +asdfghj kl +annab elle +hump back +bra ved +viswas am +multi purpose +hu midi +escor ted +barb ican +f ad +cor sa +ðŁ¤ « +pi ppa +here to +can y +ser gi +or cas +o vie +ed ou +s any +glob alization +man cini +food truck +f is +defi brill +sch re +sma fia +love wins +la ut +k aka +hol lande +game on +resurg ence +out side +olympi ad +int an +abstr action +rapi d +pal om +cal le +jas min +attack ers +swag g +mit ra +ky lo +à® ² +her mitage +gor do +e ira +so sfam +roll out +exc ite +sy nod +mer rill +c als +as sa +liveli hoods +ju ve +the black +gopack go +ant lers +alban ian +wool ly +qu iche +puri fication +are th +smar thome +ne k +all blacks +mex icans +is m +ger ms +comple xion +mar ck +u shi +ðŁIJ IJ +char l +ca stic +till erson +giuli ani +biode gradable +mal bec +bo is +ju bil +im es +r ame +gene tic +esp nu +ch ley +so ho +go pher +g sc +buu ren +cu be +bridesma ids +webin ars +to e +mani pur +viol ently +notic ias +ex changing +chi ev +replac eable +muay thai +bu ss +sp il +instal ment +div ya +cait lin +o lim +fil tering +whirl wind +sta red +prior it +pr am +pompe ii +mono logue +k ite +bu ka +âĢ¦ .. +vac cine +bre ro +woz ni +sol ent +re ferr +my rt +gridi ron +galatasar ay +fro ze +clare mont +ðŁ¥ ĥ +victori as +ssel dorf +pa stures +net neutrality +ch or +ðŁij ģ +ಠ¿ +we ho +symp tom +jo sel +in ous +dragon con +power ball +p te +four thofjuly +ec la +ear buds +where abouts +salt life +depriv ation +ch ter +wi ggle +syste m +ps st +ch az +d any +ri mo +oax aca +lanapar rilla +barcel on +melanch oly +way back +ho tro +n si +l illy +kur o +ja han +intellec t +board game +ðŁı Ĭ +sneak peek +k prc +jail s +cand el +zan zi +mor timer +star ch +ra gs +p fa +long live +k art +gir ona +cro cker +christop h +precau tions +war ship +per m +paren t +van gogh +gif ford +allegh eny +ra yn +ut m +sten cil +rec alling +pen ney +z azzle +ìĥ Ŀ +hin ds +aren as +nu ev +law ler +gu in +do this +ðŁij ķ +ì¶ķ íķĺ +we g +ti b +ri din +complex es +turbul ent +pe sos +de marcus +vall arta +sam sun +kis ses +hein rich +deport es +wil ms +ur d +then ext +inki gayo +ho wi +fir sts +carri age +clean liness +mas war +is ch +ax el +si zzle +road house +fr ans +ent ourage +co bble +boo th +benedic t +tal on +fc u +year ofthe +ray on +raider nation +fo yle +ko val +pi anos +l pg +bur mese +man ure +geo caching +cosc ino +b np +fer ra +stro phy +mar ais +ce es +legen dof +kat niss +eno ch +av ed +you know +d prk +ðŁĺ¢ ðŁĺ¢ +sp un +pro st +sor rows +cent red +ke a +gal icia +? ðŁ¤Ķ +ÑĢод а +bou chard +ðŁĴĻ ðŁĴľ +yu i +seed lings +jon ah +reco vers +ny rd +board room +su ma +my japs +tun g +sha i +ir gc +eli o +wag ons +ka shi +polic emen +john nie +ale coscino +shop ify +dot ted +de tri +va w +to fficial +in your +chal mers +trac ed +no vi +by es +ari el +nipp on +la pel +gri ez +b gs +fool ing +d ita +vijay sethu +nm wx +as ot +kr anti +hel m +ve di +sic kest +mo chi +k abo +shru bs +he red +b sp +sq m +ham r +dul kar +anth a +nr f +avoid ance +at en +publi x +be arers +nas i +ha p +h ells +ðŁĸ ¥ +ภ· +thelast jedi +oh wx +ðŁį « +wa hoo +there se +rec aps +ss nhq +bird photography +v ay +pet ti +pau lo +bel vedere +( * +gr l +du vet +c pec +sa it +por sch +meas urable +avi ators +fre mantle +bre en +on om +me and +life saving +eu ref +en don +embar as +aira sia +el is +dun kin +star magic +s ill +porto bello +ki efer +ex e +mu ted +ãģ ¦ +we thepeople +logi a +liber al +theforce awakens +min ed +haun ts +freck les +care taker +s india +âķ IJ +dev lin +list on +direction er +oh n +fi garo +em manuel +du bois +cl ones +bru ise +ðŁİĪ ðŁİī +disin fe +der matology +as r +s watch +dis comfort +tam anna +pi day +mack en +k atic +delu sional +shaw nee +gu d +al bino +p ali +din gh +cucu mbers +coffe y +anticip ating +treas ured +web summit +shel tered +sav or +pedago gy +m gs +sh ma +s bu +den ali +cam pos +bubble gum +o ir +le aps +y ler +r one +sansk rit +min t +meat less +futuri st +du de +a vel +prote sted +squ ire +z aki +sz n +har court +cycl one +bour dain +gather ings +d ant +advent urer +parag on +alt man +dd ing +ban erjee +snorkel ing +mother well +mis sy +en der +glo ws +ki wis +chick pea +por o +e fron +app t +u y +speci fied +gab by +e strada +com bos +bour bon +vin i +var un +steph ani +key words +car vings +amit abh +wr ought +tw al +re els +clu bbing +ubi quit +cri t +ambed kar +æ Ļ +prun ing +vaccin ated +boe ing +s ks +lo ona +hypno sis +edel man +pho l +he w +colo sse +mckin sey +u on +to te +sacrific ing +ox i +n ang +e mu +пÑĢи ÑĢода +m th +kers wednesday +argu ed +timel apse +ris king +regul ating +ni gh +likeli hood +cu bic +au ction +rein for +pi stor +no ses +ye l +snu ggles +pe i +jean ette +ta ku +ri th +guy z +ภŀ +y te +ver ted +pay soff +jau regui +hoo ligans +procedu ral +mi b +har dy +el eng +chec kers +all ine +the met +prou dof +keerth yofficial +collabor ator +ni u +infl icted +adv ani +re twee +memor iam +f icial +ti ghter +sal em +re viewers +br ics +ben digo +am ell +tur kish +sush maswar +paul son +pal awan +mol lie +stitch er +s burgh +ir u +hay dn +en ers +aro a +u zzi +saraj evo +hel a +apol lo +nine ty +vac a +sp on +vent u +jel ena +hei fer +avo ids +sp ine +pri ze +mar ist +re creating +me de +woo den +find lay +ro fl +n di +compreh end +yu go +y ü +to work +u fos +son ar +pi ston +recor ding +tent ative +art forsale +pel lets +fre do +ÙĪ ر +mu ses +custom ization +pro found +is ner +ide ally +si am +plan kton +cm dr +man ger +fran ken +customiz able +ठ® +walk away +swi vel +vast ly +no ton +lex a +ex moor +z as +tan te +reduc tions +lol ly +hip sters +benef ited +ë ² +ww www +mascul ine +fi ji +dre y +ph ill +ane ous +nic ol +men dez +disapp ro +ch ner +through s +shen mue +east man +ðŁIJ İ +yu ck +under tale +re ys +go beavs +eng en +c na +mer r +bir k +ãģ¨ç¹ĭãģ ĮãĤĬãģŁãģĦ +âĥ£ @ +yn na +ste ed +offen der +at um +vani shing +presi denti +love them +g nocchi +fri ggin +per il +mad hya +ag ne +dee jay +mar nock +m tb +fold able +@ ___ +stand re +bron x +bow ski +fin ite +cro ckett +b sf +ge tit +seren awilliams +mir o +ignati us +sla y +rin se +fon due +sel dom +s more +gan i +dy ce +dmit ry +cru mb +late post +pri mark +oh ana +flor als +do a +remembrance day +d ds +azi one +toon ami +air port +æĿ ± +th ad +fi st +dine sh +dr who +ad words +admi rer +pro je +kyrgy z +à « +manife station +le wan +j ic +thi bau +le ased +van ity +nouri shed +never theless +aug mente +fu elled +che ad +wil shere +ru di +p z +my co +mor ro +herbali fe +hardro ck +de man +dre ality +sp ades +ce vic +bha i +bar on +ultimat efan +hou news +to bi +stru t +ke el +affili ation +the masters +sm al +hu e +este ban +con v +om nic +datab ases +co v +ter ti +st g +snoop dogg +metab ol +leth bridge +ðŁı» âĢįâĻĢï¸ı +year ling +residente vil +nws l +iy aki +griez mann +c ous +ðŁĵĿ : +tor ian +sam i +ðŁĶ¥ðŁĶ¥ ðŁĶ¥ðŁĶ¥ðŁĶ¥ +g are +alli ances +whit field +we ther +refin ing +coy i +kra ken +ðŁĺĺ âĿ¤ +singul arity +lil i +h ns +bol dand +waw rinka +misogy ny +lo vers +c q +b dg +ad ona +gar ter +women of +sc d +recogn ising +mun a +str ou +sign alling +lare do +hell boy +alek sand +un available +pedi atric +as in +mer ia +ri shi +futuri sm +w ye +polari zed +e we +pro pel +in forms +cre ase +~ " +arti ston +like for +heidel berg +er ra +life in +len ny +inter rupt +cohe rent +ca z +vick ers +le veled +f bs +cab ins +bu mmed +apost les +we h +ten don +souven irs +infu ri +pier ce +asse t +m las +go th +di ggin +ann as +yl or +th waite +sw el +pan era +mur derers +croo ked +bs go +ac u +a on +re an +one of +ko hl +bloo dh +pest icide +lost dog +fle xing +ëĤ ĺ +su pra +eter nally +ðŁļ Ļ +pa olo +ol an +mom o +is elle +captain marvel +s lou +mistak enly +akhi lesh +mer t +il inan +bu on +bal kan +mir ro +mill en +der ail +dam on +tit i +bi os +re don +pic ard +par te +ðŁ¤ Ł +Ø º +son ics +fir sth +dd c +veg ans +tur ban +ni gan +lot tie +lyn don +star buck +pink floyd +life styles +am ara +a she +r sc +val a +sm er +cw gc +cli ent +buen as +jag an +coo ps +ðŁijij ðŁijij +speci alizes +snag ged +g lar +ben net +wildlife wednesday +bow den +pi k +art in +empor ium +ar l +re ba +pas ser +disappo ints +additi ve +âľĬ ðŁı½ +bay er +missou la +ha skell +comm ences +ni x +ne man +explo ited +plastic surgery +cc d +aso cial +vo t +sie gel +fro ome +kap am +far a +e ha +pro bes +mw f +meet ing +p bb +ak ins +mistle toe +kingdom hearts +for kids +ec r +bal e +escor ts +adidas originals +k wa +k ts +hallo ffame +ðŁĺį . +wag s +pot ted +o wing +honey comb +he fty +uro logy +mer le +b pd +stri pping +re ich +k state +gu ay +yon ge +shak ti +g loom +bat t +son om +n ery +el ba +blan ks +hel le +triple ts +bom bay +ak arta +ab ia +transm itted +rol f +ja is +angular js +fi erc +m ss +trac e +ॠĩ +tom bs +old man +kom bucha +fo l +e health +cere als +are lli +in ari +ðŁĴ © +wo l +liber ties +fa wn +af firm +nun avut +hyster ical +k drama +art es +âĢ¢âĢ¢âĢ¢âĢ¢ âĢ¢âĢ¢âĢ¢âĢ¢ +valent in +man slaughter +gal es +eo in +energi zed +del s +with draws +st les +sar castic +ram esh +incredi bles +lock hart +ya wn +ultimatefan live +oooooooo oooooooo +mu en +guru dev +te er +pe eling +new snow +lingui stics +direc tv +ag end +uni lever +ru ger +han dedly +ero se +li mel +the c +royal ties +fini shers +nr g +m gt +fid get +com ps +bac on +aggre ssively +ab it +ch â +tar de +slu gger +q anda +gre ening +d ats +ensla ved +spec tor +o ye +fre ef +b hand +stop brexit +mis conceptions +cav a +ðŁĺįðŁĺįðŁĺįðŁĺį ðŁĺįðŁĺįðŁĺįðŁĺį +multit asking +hou sel +ferre ira +cen time +ank les +jo dh +hel ly +fro me +out tuesday +nar nia +bal aji +l bloggers +jyo ti +ðŁį ĩ +lan cia +cap ri +y ap +nat ash +down fall +." âĢĶ +à ® +ligam ent +coat ings +ai ded +hi ko +fall ing +encryp ted +yeg food +infringe ment +cu di +ce p +ðŁĺį ðŁĺĤ +tra d +super rugby +ed win +wh iche +vi meo +lay ne +in vigor +he he +dubrov nik +bie ber +u tr +sham an +op ers +ham ill +en ig +di f +ar um +scrap book +min h +diver gence +mckin non +life time +guter res +wil le +ple as +patt y +mic ron +k z +dom aine +ru sher +m ds +ches ney +screw driver +âģ© , +sle dge +hau er +chan a +stam ina +sprink ler +pl n +he ff +bol ton +om on +car rington +accor dion +jor ge +inter ception +in puts +gu ll +tran scription +vanu atu +it ical +eth os +tic h +spac ey +pee king +u mi +ha ger +psycho tic +illi an +illi a +bonnar oo +an ese +pu c +laghate parth +en hall +econom ical +dre dge +% - +u we +tu bular +scoun cil +pe asants +fl er +tumb ler +he p +ford ham +row ley +initi als +ev asion +er nation +plu gins +coch ran +c attle +acid ity +ðŁİĬ ðŁİī +re grann +jump man +ef ace +x ma +patri archy +esco bar +cristi an +tip ton +nu eva +hack ney +back seat +kill arney +aid an +sta dion +simul taneous +ida ho +a je +u th +figu re +clo s +bur k +volun tar +rec ite +macfar lane +cur few +bou do +w gn +sti x +sla p +scrat ched +philli p +jour ne +ex pelled +wa z +u ke +tati ana +ou e +ho pp +dimit ri +ðŁĵ £ +mato logist +electri fying +blu ffs +bill smafia +az cardinals +y aa +x mas +shar a +r ith +g ills +dre s +bar ton +authori zation +imperi alism +home of +to do +foot path +band width +visit spain +moh sin +erup ted +mi ki +insig nia +mike l +ss h +ger a +bank holiday +aw an +t weak +star craft +e al +construc tion +skelet ons +le ep +ine m +bar clay +ship wreck +monsi eur +yo h +ron t +form ative +ser o +le p +horse man +hoo sier +haz mat +cylin ders +cen ti +ðŁĴ¥ðŁĴ¥ ðŁĴ¥ +re em +na ire +mus ically +gras shopper +est onian +termin ology +ro main +blogger rt +tox in +stan ce +cultiv ated +an ast +ðŁIJ į +shi mano +go pher +ene i +recycla ble +gam ification +fight for +c q +avoc ados +ke ys +eli ke +gly cer +shak ur +mobili zation +gal ley +expla in +ex changed +pe th +obe dience +illa ge +en nis +ãĥ ŀ +wi v +walla bies +ma ar +ig ers +fin tech +fin alized +wo j +meaning less +in field +onna ise +e et +bron te +pass ages +ðŁij § +strick land +northern lights +lom ond +h tc +wr ay +shi fter +di alog +ðŁį į +>> >>>> +te atime +ste ch +sic huan +qu ill +fran ca +comple mentary +bar rington +marcu s +mal am +goo oo +for sa +elec tra +af s +âĹ Ĩ +tri fe +sn azzy +fo lia +and olan +after dark +wood son +stra de +litt lest +o gun +con wy +co wards +ðŁĺĤðŁĺĤðŁĺĤðŁĺĤ ðŁĺĤðŁĺĤðŁĺĤ +íĬ ¸ +se ul +mur phy +dun ks +kapil shar +jo achim +wom ack +equal ity +aver ages +a ine +ðŁ¦ Ī +tac ular +dis ability +u ked +mid century +bar thol +teas ers +tab ern +nj caa +sp out +op i +ku bball +bl om +so ar +popu lism +meth yl +ðŁijĬ ðŁı¼ +o spre +alo ils +ðŁĵ ĸ +ðŁĮ ļ +x er +sp illing +publ ica +car dam +adi sh +sa cha +p kg +bu da +lyric ist +i bc +gru mp +ho ver +hal ep +anti body +anem one +âĻ¥âĻ¥ âĻ¥âĻ¥ +m cl +litho graph +cc u +s fest +path ic +calli ster +otta wa +gun sn +rut ger +hali but +en vision +differenti ate +ðŁļĢ ðŁļĢ +pir an +lat el +uc n +trou bad +ra ine +fierc ely +learn english +lea se +wex mondays +em it +dray ton +bur rell +scuba diving +hol ler +dr u +clo cked +w ral +ap ro +trans lucent +w bo +patri arch +mo ja +lan nister +fish ery +ne derland +mil dly +mi rai +ma ko +ja p +ðŁĺ©ðŁĺ© ðŁĺ© +pro statec +p anna +ar ama +under taking +tomp kins +ne op +soli ds +sav oury +e ames +cut lery +wood bridge +steam er +ri zzo +wild cat +rat na +lamin ated +kin eni +jal ap +ai des +acknowle dges +?! ?!?! +! ðŁİī +w afc +mag gio +ha ves +dar je +of i +gr il +v asi +bru x +mo hd +fake speare +arn old +r mb +for be +wal leye +ro di +therapeu tics +strate gi +ob ste +mu dder +download able +dd ings +d ca +asi angames +campe on +appropri ation +th century +ram atta +dra ped +bul lion +mu c +one x +se greg +ophel ia +bod ily +âĿ¤ ðŁĺį +wi zar +te ased +ade my +to id +sur a +lazar us +sn ickers +ma se +lo h +bow ed +bibli o +x change +har lan +gho shal +flavor ful +bha gat +alle z +whiche ver +ten stein +disc er +organ iser +mt g +dream liner +t se +hok kaido +mo k +indulg ent +hick man +blin ded +al yn +aaa ah +sp ool +lough borough +inter pret +et v +aristo tle +optimi zing +avici i +madu rai +ju li +naw az +mat chups +ab ide +paint ing +w elling +vel i +octag on +in scribed +po king +plac er +life cycle +kili g +g sp +eli ves +cle ments +na sheed +me sut +incarcer ated +dist illed +wal ang +delic acy +del gado +che z +ch ita +ad ero +tu x +pati l +o do +abh cosmetics +tv c +p bc +in accurate +hardwork paysoff +ball er +quot ation +merchandi sing +ga stri +defen ses +dro gba +bex hill +ban kno +win ona +si eg +p gs +hahah ha +agu chi +su bram +mirac le +de sch +li bre +ba cher +ent ine +bbcra di +lou dest +r ps +pi erc +fr yer +storm trooper +rafael nadal +pas co +exhau stion +epic onetsy +rc tid +kel lie +ga ines +d bz +sm riti +s bridge +lim ited +cla w +technic al +bio graphical +ado red +ภ° +exclu de +ac adia +key boards +fur man +so ca +sur u +ni ps +sw aps +server less +run e +pu ffy +north ampton +nish ings +hen der +cartri dges +gun shot +ðŁĵ ¹ +fil ament +respon dents +pey ton +mountaine er +mer ging +life span +intimid ation +p afc +nl wx +expan sive +pur r +f ck +ca e +at ti +tele thon +so hn +mend el +lo pes +dor i +un broken +te red +tast ings +in active +disin tegr +t assel +share the +pi ano +is lay +air space +z awa +ricci ardo +ming ton +fresh er +cur ry +re vs +pharo ah +h mv +exhilar ating +wh oo +lin kin +kri spy +competen cy +ste wards +ne bu +kat su +ad mins +baz ar +as ar +giving back +s summit +song z +lin us +raj kumar +farm ington +fanta sia +ðŁĺ´ ðŁĺ´ +so bri +lis se +barry more +pri sm +blo b +sen ew +mono xide +exp ire +eigh teen +di pper +xi ao +kil t +hin ch +bbc sport +bam boo +p ter +ex al +ðŁ¦ ĭ +ham lin +expe ditions +star gazing +food security +wy lie +ul f +st ingly +on storm +lo eb +bro ome +bn ha +pancre atic +eli ve +!!!!!!!! !!! +ther apper +ortho pedic +avengers endgame +antit rust +ìļ ° +go te +om d +off side +gy llen +win eries +white water +ad l +lu pita +exce eds +consi sted +chew bacca +ash leigh +nhl jets +is san +sh ld +hay at +cran berries +ðŁ¤ĺ ðŁı½ +rock the +spring training +fall out +dairy free +wa j +un decided +so wn +rc n +north wales +htt r +fu mble +d its +comp elled +popu list +min ted +blan chett +. '' +pro pulsion +m illa +au berg +her tz +h ta +u daipur +serendip ity +azte cs +als ace +ðŁIJ ij +lu n +sho es +char li +gar za +ðŁĴ Ł +pro biotics +fox tv +ol is +mi ff +loc alized +diffu ser +si gue +fun ko +rend ous +ðŁĴ ij +jeky ll diff --git a/configs/sdxl-refiner/tokenizer_2/special_tokens_map.json b/configs/sdxl-refiner/tokenizer_2/special_tokens_map.json new file mode 100644 index 000000000..ae0c5be6f --- /dev/null +++ b/configs/sdxl-refiner/tokenizer_2/special_tokens_map.json @@ -0,0 +1,24 @@ +{ + "bos_token": { + "content": "<|startoftext|>", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "pad_token": "!", + "unk_token": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + } +} diff --git a/configs/sdxl-refiner/tokenizer_2/tokenizer_config.json b/configs/sdxl-refiner/tokenizer_2/tokenizer_config.json new file mode 100644 index 000000000..a8438e020 --- /dev/null +++ b/configs/sdxl-refiner/tokenizer_2/tokenizer_config.json @@ -0,0 +1,33 @@ +{ + "add_prefix_space": false, + "bos_token": { + "__type": "AddedToken", + "content": "<|startoftext|>", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "clean_up_tokenization_spaces": true, + "do_lower_case": true, + "eos_token": { + "__type": "AddedToken", + "content": "<|endoftext|>", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + }, + "errors": "replace", + "model_max_length": 77, + "pad_token": "!", + "tokenizer_class": "CLIPTokenizer", + "unk_token": { + "__type": "AddedToken", + "content": "<|endoftext|>", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + } +} diff --git a/configs/sdxl-refiner/tokenizer_2/vocab.json b/configs/sdxl-refiner/tokenizer_2/vocab.json new file mode 100644 index 000000000..469be27c5 --- /dev/null +++ b/configs/sdxl-refiner/tokenizer_2/vocab.json @@ -0,0 +1,49410 @@ +{ + "!": 0, + "!!": 1443, + "!!!": 11194, + "!!!!": 4003, + "!!!!!!!!": 11281, + "!!!!!!!!!!!!!!!!": 30146, + "!!!!!!!!!!!": 49339, + "!!!!!!!!!!": 35579, + "!!!!!!!!!": 28560, + "!!!!!!!!": 21622, + "!!!!!!!": 15203, + "!!!!!!": 9168, + "!!!!!": 5203, + "!!!!": 2360, + "!!!\"": 28048, + "!!!)": 42532, + "!!!": 995, + "!!\"": 20556, + "!!#": 34997, + "!!)": 28352, + "!!": 748, + "!!@": 40705, + "!\"": 2947, + "!\"@": 43819, + "!#": 9670, + "!'": 13222, + "!),": 37904, + "!).": 26225, + "!)": 4571, + "!*": 37737, + "!,": 29325, + "!-": 43499, + "!...": 22121, + "!..": 35475, + "!.": 22517, + "!:)": 31671, + "!:": 17545, + "!": 256, + "!?!": 29767, + "!?!?": 47081, + "!?": 6004, + "!@": 15117, + "!]": 34466, + "!âĢ¦": 35068, + "!âĿ¤ï¸ı": 32559, + "!ðŁİī": 49085, + "!ðŁĺĬ": 43434, + "!ðŁĺį": 36438, + "\"": 1, + "\"!": 10377, + "\"\"": 41530, + "\"\"\"": 25539, + "\"\"": 8575, + "\"#": 8345, + "\"'": 31065, + "\"(": 32741, + "\")": 13112, + "\",": 4332, + "\"-": 9375, + "\"....": 37785, + "\"...": 9049, + "\"..": 25403, + "\".": 2811, + "\"/": 39486, + "\":": 7811, + "\";": 37549, + "\"": 257, + "\"?": 11727, + "\"@": 1512, + "\"@_": 20236, + "\"[": 36930, + "\"âĢ¦": 33993, + "\"âĢĶ": 41151, + "#": 2, + "##": 15483, + "#...": 31491, + "#:": 30144, + "#": 258, + "#@": 35062, + "#âĢ¦": 12834, + "#âĢİ": 34262, + "$": 3, + "$$": 24233, + "$$$": 31859, + "$$": 14929, + "$)": 39460, + "$.": 34682, + "$": 259, + "%": 4, + "%!": 35070, + "%),": 37819, + "%)": 16063, + "%,": 14505, + "%-": 48784, + "%.": 12475, + "%;": 33379, + "%": 260, + "&": 5, + "&&": 27791, + "&": 261, + "'": 6, + "'!": 13781, + "'\"": 19479, + "'#": 15319, + "''": 46594, + "''": 8445, + "')": 19175, + "',": 5662, + "'-": 26152, + "'...": 20474, + "'.": 4645, + "':": 7182, + "';": 44517, + "'": 262, + "'?": 17242, + "'@": 26397, + "'d": 1896, + "'ll": 1342, + "'m": 880, + "'re": 982, + "'s": 568, + "'t": 713, + "'ve": 1200, + "'âĢ¦": 42120, + "(": 7, + "(!)": 30253, + "(\"": 18741, + "(#": 6229, + "($)": 46597, + "($": 15186, + "(&": 15042, + "('": 18235, + "((": 22944, + "(((": 33287, + "((": 13796, + "().": 41737, + "()": 8475, + "(*": 48004, + "(*": 39575, + "(+": 12903, + "(-": 20228, + "(...": 45159, + "(.": 43055, + "(:": 8528, + "(;": 23983, + "(": 263, + "(?)": 22885, + "(@": 2181, + "(£": 33987, + "(©": 44886, + "(ðŁĵ·:": 34610, + "(ðŁĵ·": 37999, + "(ðŁĵ¸:": 44422, + "(ðŁĵ¸": 45204, + ")": 8, + ")!!": 47518, + ")!": 7805, + ")\"": 13046, + ")#": 39981, + ")'": 23613, + ")(": 27956, + "))": 13720, + "))))": 42911, + "))))": 34181, + ")))": 18305, + "))": 5167, + "),": 2361, + ")-": 19034, + ")...": 15274, + ")..": 41822, + ").": 1818, + ")/": 26616, + "):": 4143, + ");": 19686, + ")": 264, + ")?": 18765, + ")@": 41928, + ")_/": 45028, + ")_/¯": 45781, + ")âĢ¦": 41844, + "*": 9, + "*)": 30956, + "**": 9825, + "****": 21326, + "********": 42974, + "*****": 43571, + "****": 25167, + "***": 7829, + "**": 4441, + "*,": 41895, + "*-*": 23568, + "*.": 31304, + "*": 265, + "*_*": 44535, + "+": 10, + "+)": 34810, + "++": 47298, + "+++": 35986, + "++": 19056, + "+,": 35885, + "+.": 25238, + "+/-": 47614, + "+": 266, + ",": 11, + ",\"": 3823, + ",#": 11215, + ",&": 26905, + ",'": 10599, + ",)": 44493, + ",,": 21340, + ",,,,": 33225, + ",,,": 14811, + ",,": 8844, + ",-": 29821, + ",...": 20365, + ",.": 41277, + ",": 267, + ",@": 13975, + ",âĢ¦": 14601, + "-": 12, + "-\"": 18646, + "-#": 10151, + "-$": 24946, + "-'": 28010, + "-(": 33345, + "-)": 3535, + "-*": 21527, + "--": 2154, + "----": 5753, + "--------": 11772, + "----------------": 23122, + "----": 30164, + "---->": 35999, + "---": 11079, + "--->": 14518, + "--": 2432, + "-->": 6422, + "-->>": 47252, + "-.-": 32765, + "-...": 43147, + "-.": 44040, + "-": 268, + "->": 5081, + "-@": 10087, + "-_-": 27227, + "-__": 42718, + "-âĢ¦": 30047, + ".": 13, + ".!!": 37805, + ".!": 14030, + ".\"": 18650, + ".\"-": 21234, + ".\"": 1081, + ".\"âĢĶ": 48703, + ".#": 5014, + ".'\"": 41558, + ".''": 49379, + ".'": 5938, + ".(": 22294, + ".)": 5376, + ".*": 26145, + ".,": 5276, + ".-": 12481, + "..": 608, + "..!!": 23707, + "..!": 17994, + "..\"": 15229, + "..#": 15735, + "..,": 47143, + "...": 3002, + "...!!!": 38351, + "...!!": 39915, + "...!": 16860, + "...\"": 5240, + "...#": 8195, + "...&": 44979, + "...'": 23167, + "...(": 37981, + "...)": 14040, + "...,": 42717, + "....": 2386, + "....\"": 26689, + "....#": 20346, + ".....": 34151, + ".....#": 38867, + "........": 8246, + "................": 24855, + "............": 42965, + "...........": 35008, + "..........": 25526, + ".........": 19881, + "........": 14720, + ".......": 9917, + "......": 5590, + ".....": 3104, + "....": 1390, + "....@": 29790, + "...:": 34570, + "...": 678, + "...?": 16388, + "...@": 12672, + "..": 852, + "..?": 23875, + "..@": 21124, + "./": 31975, + ".:": 15811, + ".;": 47596, + ".": 269, + ".<": 29442, + ".?": 29294, + ".@": 1230, + ".]": 33511, + ".~": 42651, + ".âĢ¦": 18047, + ".âĿ¤ï¸ı": 39085, + ".âłĢ": 30097, + ".ðŁĺĤ": 46580, + "/": 14, + "/#": 13217, + "/$": 36266, + "/-": 19811, + "/.": 39382, + "//": 15348, + "////": 46271, + "///": 22734, + "//": 3502, + "/": 270, + "/@": 8216, + "0": 15, + "0": 271, + "1": 16, + "1": 272, + "2": 17, + "2": 273, + "3": 18, + "3": 274, + "4": 19, + "4": 275, + "5": 20, + "5": 276, + "6": 21, + "6": 277, + "7": 22, + "7": 278, + "8": 23, + "8": 279, + "9": 24, + "9": 280, + ":": 25, + ":\"": 29498, + ":\")": 46432, + ":\"": 12089, + ":#": 26625, + ":$": 33769, + ":'": 8017, + ":'(": 21250, + ":')": 10701, + ":'": 23851, + ":((": 42496, + ":(": 5965, + ":)": 11070, + ":))))": 42339, + ":)))": 21840, + ":))": 10164, + ":).": 39010, + ":)": 1408, + ":*": 12617, + ":-": 13021, + ":-(": 25137, + ":-)": 4223, + ":-": 10323, + ":...": 42140, + "://": 12441, + ":/": 13604, + "::": 33077, + ":::": 43818, + "::": 9788, + ":": 281, + ":>": 39677, + ":@": 14339, + ":]": 43486, + ":|": 45986, + ":âĢ¦": 22365, + ";": 26, + ";))": 41873, + ";)": 3661, + ";-": 35657, + ";-)": 10475, + ";;": 34824, + ";;": 24492, + ";": 282, + "<": 27, + "<-": 47280, + "": 34308, + "<<": 24588, + "<": 283, + "<<": 16482, + "<<<": 35054, + "<|endoftext|>": 49407, + "<|startoftext|>": 49406, + "=": 28, + "=))": 39587, + "=)": 17840, + "=": 284, + "==": 11748, + "====": 21734, + "========": 38952, + "==>": 29688, + "=>": 9714, + ">": 29, + ">.<": 38507, + ">:": 36196, + ">": 285, + "><": 28015, + ">>": 8270, + ">>": 2988, + ">>>": 6395, + ">>>>": 18461, + ">>>>": 18435, + ">>>>>": 32972, + ">>>>>>": 48947, + ">>>>>>>>": 41947, + ">_": 44144, + "?": 30, + "?!": 9785, + "?!!": 25342, + "?!\"": 29315, + "?!": 2835, + "?!?!": 16349, + "?!?!?!": 49084, + "?!?!?": 37619, + "?!?": 11395, + "?\"": 3283, + "?#": 24018, + "?'": 13610, + "?)": 9626, + "?,": 41628, + "?...": 22641, + "?..": 43905, + "?.": 41251, + "?:": 21067, + "?": 286, + "??": 5195, + "??!!": 43219, + "??!": 37341, + "??\"": 44996, + "??": 2197, + "???": 40017, + "???": 3824, + "????": 15936, + "????": 10362, + "?????": 21370, + "??????": 34589, + "????????": 45091, + "?@": 29258, + "?ðŁ¤Ķ": 47928, + "@": 31, + "@#": 39397, + "@.": 43730, + "@/": 28639, + "@": 287, + "@@": 30314, + "@_": 2692, + "@__": 17042, + "@___": 48308, + "A": 32, + "A": 288, + "B": 33, + "B": 289, + "C": 34, + "C": 290, + "D": 35, + "D": 291, + "E": 36, + "E": 292, + "F": 37, + "F": 293, + "G": 38, + "G": 294, + "H": 39, + "H": 295, + "I": 40, + "I": 296, + "J": 41, + "J": 297, + "K": 42, + "K": 298, + "L": 43, + "L": 299, + "M": 44, + "M": 300, + "N": 45, + "N": 301, + "O": 46, + "O": 302, + "P": 47, + "P": 303, + "Q": 48, + "Q": 304, + "R": 49, + "R": 305, + "S": 50, + "S": 306, + "T": 51, + "T": 307, + "U": 52, + "U": 308, + "V": 53, + "V": 309, + "W": 54, + "W": 310, + "X": 55, + "X": 311, + "Y": 56, + "Y": 312, + "Z": 57, + "Z": 313, + "[": 58, + "[#": 11115, + "[...": 39975, + "[...]": 43790, + "[": 314, + "[@": 15148, + "[]": 22240, + "\\": 59, + "\\'": 41239, + "\\": 315, + "]": 60, + "]\"": 39434, + "],": 34067, + "].": 26262, + "]:": 21641, + "]": 316, + "][#": 39009, + "][": 29329, + "^": 61, + "^)": 30720, + "^-": 43516, + "^.": 31552, + "^.^": 35791, + "^": 317, + "^^": 34454, + "^^": 9064, + "^_": 14423, + "^_^": 15995, + "_": 62, + "_'": 44701, + "_(": 36951, + "_)": 37393, + "_*": 36237, + "_,": 31417, + "_-": 23193, + "_.": 26841, + "_/": 37647, + "_:": 13109, + "_": 318, + "__": 2355, + "__:": 47043, + "__": 3838, + "___": 43812, + "___": 13530, + "____": 4727, + "____": 25350, + "_____": 38803, + "________": 9549, + "________________": 20115, + "`": 63, + "`": 319, + "a": 64, + "a": 320, + "aa": 1821, + "aa": 3894, + "aaa": 14376, + "aaa": 9583, + "aaaa": 6727, + "aaaa": 19336, + "aaaaa": 31095, + "aaaaaa": 44413, + "aaaaaaaa": 23126, + "aaaah": 49151, + "aaah": 35856, + "aaay": 37846, + "aab": 34108, + "aac": 23251, + "aac": 11346, + "aad": 20464, + "aad": 35894, + "aaf": 37638, + "aaf": 31534, + "aag": 42174, + "aah": 28990, + "aaj": 28727, + "aaj": 43411, + "aak": 37739, + "aal": 22268, + "aal": 30208, + "aali": 27896, + "aaliyah": 46577, + "aam": 12943, + "aam": 22775, + "aama": 45018, + "aamaadmi": 45563, + "aamaadmiparty": 46406, + "aamir": 27456, + "aan": 20705, + "aan": 13426, + "aand": 38054, + "aap": 12023, + "aap": 12052, + "aapl": 34516, + "aar": 4695, + "aar": 13234, + "aard": 46932, + "aaron": 13948, + "aaron": 7709, + "aas": 28542, + "aas": 32205, + "aat": 34018, + "aat": 35004, + "aau": 35426, + "aay": 38281, + "aay": 40249, + "aaz": 26770, + "ab": 596, + "ab": 3937, + "aba": 44204, + "aba": 11102, + "abad": 33444, + "abad": 7155, + "aban": 41662, + "aband": 8595, + "abandon": 28805, + "abandoned": 11227, + "abar": 17860, + "abar": 39805, + "abas": 25402, + "abay": 43542, + "abb": 38954, + "abb": 38297, + "abba": 30870, + "abbas": 37494, + "abbas": 24412, + "abbey": 31927, + "abbey": 10132, + "abbie": 39949, + "abbo": 13536, + "abbot": 44046, + "abbott": 43737, + "abbott": 15649, + "abbrevi": 44843, + "abby": 30586, + "abby": 14694, + "abc": 13137, + "abc": 5334, + "abcnews": 31566, + "abd": 44093, + "abdel": 46511, + "abdomin": 35335, + "abdominal": 39328, + "abdu": 13361, + "abduc": 17884, + "abducted": 31520, + "abduction": 36984, + "abdul": 14227, + "abdul": 15593, + "abdullah": 21317, + "abe": 15856, + "abe": 12734, + "abee": 36037, + "abel": 31938, + "abel": 25318, + "abella": 46156, + "aben": 40865, + "aber": 7828, + "aber": 41867, + "aberdeen": 30539, + "aberdeen": 17236, + "abh": 27484, + "abh": 33649, + "abhcosmetics": 49189, + "abhi": 18113, + "abhin": 44045, + "abhishek": 44502, + "abi": 16867, + "abi": 14161, + "abia": 48604, + "abide": 49163, + "abig": 20863, + "abigail": 25686, + "abil": 21135, + "abilities": 8724, + "ability": 35146, + "ability": 3024, + "abit": 48668, + "ablanc": 33716, + "able": 10102, + "able": 863, + "abled": 10655, + "ableg": 24055, + "ables": 8486, + "ableton": 47169, + "ably": 6748, + "abnormal": 40934, + "abo": 2889, + "abo": 21861, + "aboard": 11661, + "abol": 31768, + "abolic": 46827, + "abolish": 47403, + "aboo": 42433, + "abor": 8416, + "aboriginal": 20422, + "abortion": 12336, + "abortions": 43218, + "aboss": 46401, + "abou": 36455, + "abou": 44053, + "abound": 41037, + "abour": 46637, + "about": 20204, + "about": 781, + "abouts": 36339, + "above": 35019, + "above": 4348, + "aboy": 37077, + "abpoli": 44779, + "abq": 38767, + "abr": 44932, + "abra": 10694, + "abra": 35087, + "abraham": 40623, + "abraham": 15869, + "abram": 33255, + "abrams": 29852, + "abre": 22472, + "abre": 46756, + "abri": 28605, + "abridged": 45333, + "abroad": 11253, + "abru": 46295, + "abs": 18431, + "abs": 11109, + "absc": 25389, + "abscbn": 44260, + "abscbn": 45810, + "absen": 32453, + "absence": 19240, + "absent": 30363, + "absol": 4624, + "absolu": 7055, + "absolut": 4666, + "absolute": 7501, + "absolutely": 4703, + "absor": 14303, + "absorb": 35806, + "absorbed": 45059, + "absorbing": 46412, + "absorption": 42210, + "abstr": 7530, + "abstract": 23885, + "abstract": 10197, + "abstractart": 31170, + "abstraction": 47696, + "abstracts": 40065, + "absur": 21639, + "absurd": 29757, + "abt": 9850, + "abu": 9167, + "abu": 11787, + "abud": 20180, + "abudha": 21450, + "abudhabi": 25256, + "abuja": 23371, + "abun": 20544, + "abundance": 23236, + "abundant": 31611, + "abur": 23377, + "aburger": 46660, + "abuse": 7678, + "abused": 23855, + "abuses": 37132, + "abusing": 36558, + "abusive": 26858, + "abv": 34172, + "aby": 16342, + "aby": 31378, + "abyss": 33632, + "abz": 42292, + "ac": 546, + "ac": 2816, + "aca": 9213, + "acab": 41388, + "acacia": 44047, + "acad": 32537, + "acade": 2892, + "academia": 22662, + "academic": 31178, + "academic": 7935, + "academics": 26417, + "academies": 42569, + "academy": 29968, + "academy": 4041, + "acadi": 41455, + "acadia": 49236, + "acam": 26172, + "acan": 42227, + "acan": 26318, + "acap": 32357, + "acar": 22232, + "acare": 16961, + "acc": 26805, + "acc": 9318, + "acca": 30883, + "acce": 8564, + "acceler": 10161, + "accelerate": 23619, + "accelerated": 38513, + "accelerating": 41821, + "acceleration": 39387, + "accelerator": 25261, + "accent": 28110, + "accent": 18931, + "accents": 31738, + "accenture": 41853, + "accep": 4616, + "accept": 16447, + "accept": 9338, + "acceptable": 14209, + "acceptance": 17090, + "accepted": 9159, + "accepting": 12855, + "accepts": 22338, + "access": 7596, + "access": 3822, + "accessi": 10787, + "accessibility": 23407, + "accessible": 13977, + "accessing": 46339, + "accessories": 10220, + "accessory": 20417, + "acci": 4263, + "acci": 33943, + "accident": 6608, + "accidental": 24895, + "accidentally": 11061, + "accidents": 22072, + "acclaimed": 21172, + "acco": 44730, + "accol": 33858, + "accolades": 46731, + "accom": 23658, + "accommo": 34495, + "accommod": 14386, + "accommodate": 34708, + "accommodation": 18066, + "accommodations": 45536, + "accomp": 24985, + "accompan": 14746, + "accompanied": 20715, + "accompany": 34142, + "accompanying": 38179, + "accompli": 10205, + "accomplish": 25542, + "accomplished": 16462, + "accomplishment": 26100, + "accomplishments": 24965, + "accor": 4182, + "accord": 34293, + "accord": 28513, + "according": 4717, + "accordingly": 35535, + "accordion": 48760, + "accoun": 3081, + "account": 18424, + "account": 4684, + "accountability": 19377, + "accountable": 24216, + "accountant": 31026, + "accountants": 37222, + "accounted": 43951, + "accounting": 14805, + "accounts": 9974, + "accra": 31900, + "accred": 17451, + "accreditation": 27015, + "accredited": 27647, + "acct": 45569, + "accu": 5618, + "accumul": 19275, + "accumulation": 37112, + "accur": 6551, + "accuracy": 18423, + "accurate": 8858, + "accurately": 24206, + "accusations": 33615, + "accuse": 39414, + "accused": 9434, + "accuses": 27496, + "accusing": 41474, + "acdc": 45067, + "ace": 2675, + "ace": 804, + "acea": 35219, + "aceae": 38153, + "acele": 40868, + "aceous": 33610, + "acer": 37990, + "acer": 25809, + "aces": 5725, + "acet": 28735, + "acf": 38389, + "ach": 972, + "ach": 987, + "acha": 22686, + "acharya": 45780, + "achat": 32706, + "ache": 27771, + "ache": 7214, + "ached": 17048, + "acher": 38442, + "acher": 17936, + "achers": 25051, + "aches": 14823, + "achi": 3264, + "achi": 9087, + "achiev": 8160, + "achieve": 14798, + "achieve": 8175, + "achieved": 12359, + "achievement": 8245, + "achievements": 16114, + "achiever": 46286, + "achievers": 44544, + "achieves": 40123, + "achieving": 16120, + "achilles": 33327, + "achim": 42335, + "aching": 12864, + "acho": 33130, + "achs": 41195, + "aci": 4359, + "aci": 34100, + "acia": 30163, + "acial": 32422, + "acid": 35474, + "acid": 10085, + "acidity": 48800, + "acids": 27751, + "acies": 20162, + "acin": 39442, + "acing": 9442, + "acio": 26202, + "acion": 44965, + "acion": 24968, + "acional": 26435, + "aciones": 35832, + "acious": 16020, + "acity": 7511, + "ación": 38175, + "ack": 877, + "ack": 725, + "acked": 5698, + "acker": 31201, + "acker": 7940, + "ackeray": 41843, + "acki": 42857, + "acking": 5515, + "ackles": 28503, + "acknow": 13563, + "acknowle": 18100, + "acknowledge": 25209, + "acknowledged": 35913, + "acknowledges": 49083, + "acknowledging": 45645, + "acks": 3858, + "acl": 47593, + "acl": 23073, + "acle": 6504, + "acles": 34164, + "aclu": 37354, + "acm": 39317, + "acmilan": 36500, + "acne": 24195, + "aco": 9463, + "aco": 8800, + "acol": 17431, + "acollege": 43468, + "acom": 17224, + "acom": 22342, + "acon": 11621, + "acon": 11571, + "aconf": 38851, + "acons": 31599, + "acor": 22076, + "acorn": 37537, + "acos": 39943, + "acosta": 31994, + "acou": 8794, + "acoun": 31295, + "acounty": 45449, + "acoustic": 10616, + "acoustics": 43873, + "acp": 19627, + "acqu": 7946, + "acquainted": 40713, + "acqui": 12194, + "acquire": 21576, + "acquired": 15932, + "acquires": 27376, + "acquiring": 42785, + "acquis": 14207, + "acquisition": 16543, + "acquisitions": 39649, + "acr": 43648, + "acre": 26749, + "acre": 9493, + "acres": 11630, + "acro": 21060, + "acrob": 40891, + "acron": 37770, + "across": 2500, + "acrosse": 40979, + "acruz": 40455, + "acry": 10440, + "acrylic": 12252, + "acs": 11782, + "act": 10305, + "act": 1393, + "acted": 10971, + "acti": 4786, + "acting": 6319, + "action": 12493, + "action": 1816, + "actions": 6271, + "activ": 3430, + "activate": 26737, + "activated": 22249, + "activation": 26769, + "active": 19009, + "active": 4046, + "actively": 18645, + "activi": 7230, + "activism": 20117, + "activist": 10850, + "activists": 12649, + "activities": 6514, + "activity": 6206, + "actment": 44807, + "acton": 36167, + "acton": 36697, + "actonclimate": 43797, + "actor": 12181, + "actor": 4035, + "actors": 9255, + "actorslife": 25117, + "actorvijay": 34033, + "actress": 5805, + "actresses": 33639, + "acts": 6816, + "actu": 2375, + "actual": 7488, + "actually": 2955, + "acu": 9204, + "acu": 48475, + "aculture": 38145, + "acup": 30869, + "acup": 27278, + "acupuncture": 40043, + "acur": 44719, + "acura": 30120, + "acus": 33710, + "acute": 19734, + "acy": 18717, + "acy": 2356, + "ad": 594, + "ad": 680, + "ada": 25785, + "ada": 1886, + "adaily": 47254, + "adal": 46646, + "adam": 6037, + "adam": 4944, + "adamlambert": 27659, + "adams": 7942, + "adan": 41802, + "adani": 37499, + "adap": 6341, + "adapt": 22666, + "adaptation": 16566, + "adapted": 26657, + "adapter": 21839, + "adapting": 44120, + "adaptive": 28672, + "adar": 27702, + "adar": 32681, + "adas": 23250, + "adata": 39500, + "aday": 31367, + "aday": 10280, + "adays": 24337, + "adb": 45630, + "adc": 38201, + "add": 19408, + "add": 3536, + "addams": 38912, + "added": 4149, + "adder": 47557, + "addi": 36378, + "addic": 5709, + "addict": 14614, + "addicted": 16275, + "addiction": 11751, + "addictive": 29638, + "addicts": 29997, + "adding": 8676, + "addis": 43911, + "addison": 32369, + "additi": 26927, + "addition": 6698, + "additional": 10666, + "additions": 22575, + "additive": 48546, + "addo": 40001, + "address": 5834, + "addressed": 20817, + "addresses": 12702, + "addressing": 10594, + "adds": 9944, + "addy": 24746, + "ade": 2194, + "ade": 1928, + "adecides": 46374, + "aded": 9994, + "adee": 47054, + "adel": 4434, + "adel": 27308, + "adelaide": 38193, + "adelaide": 11611, + "adele": 42843, + "adele": 21220, + "adelrey": 43627, + "ademy": 49123, + "aden": 28669, + "aden": 28688, + "adena": 23648, + "adequ": 18232, + "adequate": 22281, + "ader": 21365, + "adero": 49185, + "aders": 27672, + "ades": 5793, + "adh": 42301, + "adhd": 32649, + "adhe": 21175, + "adhesive": 38429, + "adi": 2486, + "adi": 8779, + "adia": 26874, + "adic": 36780, + "adid": 8086, + "adidas": 22396, + "adidas": 9589, + "adidasoriginals": 48575, + "adies": 45834, + "adifference": 37217, + "adilla": 41167, + "ading": 15000, + "adio": 15060, + "adirond": 36843, + "adish": 49009, + "adity": 28596, + "aditya": 37186, + "adityanath": 44437, + "adjac": 32517, + "adjacent": 33836, + "adjec": 45512, + "adju": 16413, + "adjun": 45995, + "adjust": 13784, + "adjust": 28073, + "adjustable": 20476, + "adjusted": 30515, + "adjusting": 41132, + "adjustment": 36081, + "adjustments": 36331, + "adl": 49351, + "adler": 30222, + "adm": 9892, + "adm": 33604, + "admi": 11666, + "admin": 12528, + "admini": 6434, + "administr": 12174, + "administration": 9502, + "administrative": 22424, + "administrator": 22603, + "administrators": 36123, + "admins": 49297, + "admir": 17031, + "admiral": 21013, + "admiration": 39569, + "admire": 17791, + "admired": 36103, + "admirer": 48344, + "admiring": 29835, + "admission": 11315, + "admissions": 22463, + "admit": 13769, + "admits": 16332, + "admitted": 20427, + "admitting": 46148, + "adn": 40339, + "adnan": 42037, + "ado": 4775, + "ado": 2933, + "adobe": 29256, + "adobe": 16484, + "adog": 44913, + "adol": 33512, + "adole": 22704, + "adolescent": 36793, + "adolescents": 45656, + "adolf": 41179, + "adon": 25907, + "adona": 48419, + "adop": 4183, + "adopt": 16441, + "adopt": 11159, + "adoptable": 36905, + "adoptdont": 19674, + "adoptdontshop": 20089, + "adopted": 12538, + "adopting": 30158, + "adoption": 11544, + "adopts": 40853, + "ador": 4992, + "ador": 9162, + "adora": 40031, + "adorable": 6298, + "adoration": 46781, + "adore": 15502, + "adored": 49233, + "adores": 30290, + "adorned": 44953, + "ados": 20079, + "adox": 32188, + "adp": 44426, + "adr": 46189, + "adren": 24204, + "adrenaline": 35552, + "adri": 5935, + "adrian": 25012, + "adrian": 13163, + "adriana": 41363, + "adrid": 26562, + "adrien": 47469, + "adrienne": 40081, + "ads": 2485, + "adu": 16882, + "adu": 24446, + "adukone": 30511, + "adul": 7222, + "adult": 42209, + "adult": 7115, + "adulthood": 40964, + "adults": 9391, + "adv": 1647, + "adv": 21018, + "advan": 33411, + "advance": 27291, + "advance": 7022, + "advanced": 7465, + "advancement": 35437, + "advances": 15852, + "advancing": 21355, + "advani": 48189, + "advant": 7017, + "advantage": 8573, + "advantaged": 38361, + "advantages": 23506, + "adven": 41670, + "advent": 3071, + "advent": 15199, + "adventcalendar": 43492, + "adventur": 29627, + "adventure": 17251, + "adventure": 4377, + "adventurer": 48098, + "adventures": 7941, + "adventurous": 31179, + "adver": 4806, + "adverse": 30348, + "adversity": 32516, + "advert": 19080, + "adverti": 5682, + "advertise": 31473, + "advertised": 38987, + "advertisement": 18713, + "advertiser": 41829, + "advertisers": 45472, + "advertising": 8158, + "adverts": 44306, + "advice": 4973, + "advis": 4634, + "advise": 25962, + "advised": 23196, + "adviser": 20367, + "advisers": 40984, + "advises": 42761, + "advising": 39648, + "advisor": 12380, + "advisors": 23197, + "advisory": 10224, + "advoc": 6657, + "advocacy": 14443, + "advocate": 12044, + "advocates": 17757, + "adwords": 48343, + "ady": 41446, + "ady": 8781, + "ae": 5548, + "ae": 4542, + "aea": 37048, + "aed": 26912, + "aege": 42304, + "ael": 41533, + "ael": 43340, + "aen": 43085, + "aer": 10195, + "aeri": 27685, + "aerial": 44866, + "aerial": 12440, + "aero": 10196, + "aero": 25026, + "aerob": 42824, + "aeron": 37286, + "aeronau": 42816, + "aerop": 27735, + "aerosmith": 43253, + "aerospace": 20530, + "aes": 10617, + "aes": 35677, + "aest": 40694, + "aesthe": 21181, + "aesthetic": 16179, + "aesthetics": 29295, + "aew": 47108, + "af": 702, + "af": 4391, + "afa": 24953, + "afan": 47474, + "afar": 41637, + "afar": 37866, + "afb": 27022, + "afc": 29742, + "afc": 6571, + "afcb": 44276, + "afcon": 30019, + "afd": 44626, + "afe": 30487, + "afe": 13912, + "afer": 44707, + "aff": 8849, + "aff": 14864, + "affair": 13998, + "affairs": 9830, + "affe": 4556, + "affect": 11361, + "affected": 9715, + "affecting": 18448, + "affection": 33780, + "affection": 28381, + "affectionate": 42578, + "affects": 17285, + "affili": 12120, + "affiliate": 18652, + "affiliated": 37540, + "affiliation": 48377, + "affinity": 41451, + "affir": 25343, + "affirm": 42711, + "affirm": 48625, + "affirmation": 47495, + "affl": 34036, + "affleck": 35584, + "afford": 7951, + "afford": 13223, + "affordability": 44828, + "affordable": 43944, + "affordable": 8926, + "afg": 33994, + "afgh": 9029, + "afghan": 15919, + "afghanistan": 9836, + "afi": 24074, + "afi": 31958, + "afil": 27209, + "afire": 42010, + "afirst": 38601, + "afl": 15132, + "afl": 14356, + "aflo": 41959, + "afm": 38385, + "afootball": 41694, + "afor": 43102, + "afore": 41468, + "afp": 18311, + "afraid": 9474, + "afri": 13888, + "afric": 2136, + "africa": 3093, + "african": 17471, + "african": 4736, + "africans": 26534, + "afridi": 37651, + "afrika": 45833, + "afrin": 45586, + "afro": 16267, + "afro": 21795, + "afs": 48960, + "aft": 22693, + "after": 2278, + "after": 953, + "afterdark": 48966, + "afterlife": 46790, + "aftermath": 20958, + "afterno": 22330, + "afternoon": 39035, + "afternoon": 2716, + "afternoons": 31631, + "afterparty": 35305, + "afterwards": 23911, + "ag": 602, + "ag": 5241, + "aga": 1050, + "aga": 4654, + "again": 1495, + "against": 23838, + "against": 1601, + "agame": 46943, + "agan": 42946, + "agan": 9178, + "agar": 13199, + "agar": 17544, + "agarwal": 43117, + "agas": 20430, + "agate": 25454, + "agatha": 43896, + "agave": 42671, + "agawa": 39433, + "agazine": 44942, + "age": 4758, + "age": 805, + "aged": 3889, + "ageing": 25349, + "agen": 10101, + "agen": 43696, + "agencies": 13887, + "agency": 44885, + "agency": 6270, + "agend": 48653, + "agenda": 8728, + "agent": 21210, + "agent": 6576, + "agents": 10199, + "agentsof": 37074, + "agentsofshield": 38801, + "ager": 44847, + "ager": 10443, + "agers": 22123, + "ages": 2321, + "agg": 45482, + "aggarwal": 39386, + "agger": 27836, + "aggi": 36844, + "aggie": 44244, + "aggie": 37618, + "aggies": 31047, + "aggio": 36685, + "aggrav": 35203, + "aggre": 10426, + "aggreg": 41968, + "aggregate": 41318, + "aggression": 28900, + "aggressive": 16295, + "aggressively": 48667, + "agh": 17917, + "agh": 14402, + "aghan": 31276, + "agi": 24036, + "agi": 17645, + "agic": 37652, + "agile": 16276, + "agility": 32161, + "aging": 4336, + "agio": 41746, + "agirl": 35469, + "agle": 37035, + "agle": 16702, + "agles": 36374, + "agles": 22679, + "aglia": 46912, + "agm": 19162, + "agn": 36474, + "agna": 43626, + "agne": 29374, + "agne": 48303, + "agnes": 26213, + "agno": 41540, + "ago": 6276, + "ago": 1468, + "agomez": 27127, + "agon": 26775, + "agon": 14901, + "agony": 36977, + "agor": 38920, + "agos": 32657, + "agov": 34227, + "agp": 46048, + "agr": 36639, + "agra": 26660, + "agra": 29830, + "agram": 2447, + "agre": 3180, + "agreat": 37594, + "agree": 5953, + "agreed": 12774, + "agreeing": 40720, + "agreement": 8286, + "agreements": 25865, + "agrees": 17854, + "agri": 20527, + "agri": 30326, + "agricul": 7234, + "agricultural": 15440, + "agriculture": 9720, + "agro": 33178, + "agro": 44589, + "agron": 41314, + "agroup": 40099, + "ags": 16926, + "agt": 39681, + "agu": 3922, + "agu": 36544, + "agua": 18482, + "aguchi": 49206, + "ague": 2095, + "aguero": 42964, + "agues": 7000, + "aguil": 27946, + "aguilar": 44715, + "ah": 1772, + "ah": 1288, + "aha": 12082, + "aha": 8429, + "ahah": 38661, + "ahaha": 32423, + "ahahaha": 42620, + "aham": 36036, + "ahan": 45061, + "ahan": 19255, + "ahar": 31038, + "ahar": 38760, + "ahe": 27688, + "ahead": 3158, + "ahem": 39995, + "ahh": 13152, + "ahhh": 14769, + "ahhhh": 21054, + "ahhhhh": 36392, + "ahi": 45349, + "ahi": 24154, + "ahl": 30433, + "ahmad": 32167, + "ahmad": 16902, + "ahmadi": 38656, + "ahmadiyya": 44865, + "ahmed": 19491, + "ahmed": 12081, + "ahmedabad": 26966, + "ahn": 33405, + "aho": 28114, + "aho": 38444, + "ahora": 43113, + "ahouse": 33197, + "ahoy": 38652, + "ahs": 16937, + "ahu": 11908, + "ahu": 16515, + "ai": 2014, + "ai": 2215, + "aia": 27046, + "aib": 34780, + "aic": 29454, + "aid": 13723, + "aid": 5182, + "aida": 33830, + "aidan": 48814, + "aidan": 26945, + "aide": 31558, + "aide": 9746, + "aided": 48707, + "aiden": 40020, + "aides": 49082, + "aids": 11759, + "aig": 27295, + "aig": 46989, + "aii": 22478, + "aik": 42575, + "aiken": 46342, + "ail": 1457, + "ail": 9154, + "ailed": 38919, + "ailing": 29999, + "ails": 27024, + "aim": 6787, + "aim": 11255, + "aime": 39872, + "aimed": 20247, + "aimee": 36318, + "aiming": 21768, + "aimo": 36706, + "aims": 13326, + "ain": 8326, + "ain": 2210, + "aine": 48983, + "aine": 17634, + "ains": 27621, + "aint": 29543, + "aint": 13099, + "ainted": 39933, + "aioli": 43949, + "air": 1281, + "air": 1922, + "aira": 35085, + "aira": 46444, + "airasia": 48020, + "airbnb": 23098, + "airborne": 22755, + "airbus": 15324, + "aircraft": 7706, + "airdrop": 38434, + "aire": 7682, + "aired": 21938, + "aires": 17034, + "airfield": 40525, + "airforce": 23511, + "airing": 20453, + "airline": 14847, + "airlines": 8929, + "airmen": 44499, + "airplane": 16451, + "airplanes": 33319, + "airplay": 47024, + "airpollution": 47362, + "airport": 48337, + "airport": 3259, + "airports": 21543, + "airs": 18539, + "airshow": 27139, + "airsoft": 30134, + "airspace": 49280, + "airstrikes": 37220, + "airtel": 34784, + "airtime": 46617, + "airwaves": 43910, + "airways": 14299, + "airy": 44453, + "ais": 7616, + "ais": 11393, + "aise": 30505, + "aish": 21946, + "aisha": 40211, + "aishwar": 29687, + "aishwarya": 44019, + "aisle": 26917, + "ait": 25613, + "ait": 40814, + "aj": 3990, + "aj": 6342, + "aja": 42343, + "aja": 19633, + "ajax": 21933, + "ajay": 22494, + "ajay": 28726, + "ajaydevgn": 35515, + "aje": 48818, + "aje": 33315, + "ajes": 38791, + "aji": 26102, + "aji": 21153, + "ajit": 42261, + "ajith": 24118, + "ajo": 26958, + "aju": 36855, + "ak": 819, + "ak": 1196, + "aka": 19154, + "aka": 3412, + "akaif": 45736, + "akan": 43678, + "akan": 38244, + "akapoor": 40064, + "akarta": 48603, + "akb": 41962, + "akbar": 27180, + "ake": 10558, + "ake": 5776, + "aked": 6115, + "aker": 14245, + "aker": 3074, + "akers": 5788, + "akes": 4764, + "akest": 46679, + "akh": 14821, + "akh": 30660, + "akhan": 28158, + "akhi": 41660, + "akhilesh": 48495, + "akhtar": 45458, + "aki": 18173, + "aki": 6592, + "akin": 24630, + "akin": 13601, + "aking": 1809, + "akins": 48568, + "akira": 34001, + "akis": 27732, + "akistan": 46221, + "akley": 39908, + "ako": 44027, + "ako": 14541, + "akon": 47105, + "akos": 44659, + "akrish": 37434, + "akron": 26115, + "aks": 2953, + "aksh": 28226, + "akshay": 21483, + "akshay": 38914, + "akshaykumar": 23624, + "akshi": 42634, + "aku": 18151, + "aku": 20815, + "aky": 11977, + "al": 526, + "al": 566, + "ala": 12783, + "ala": 3449, + "alab": 6365, + "alabam": 45880, + "alabama": 8422, + "alach": 24622, + "alad": 23074, + "aladdin": 29951, + "alai": 47072, + "alain": 28999, + "alam": 16612, + "alam": 16012, + "alamo": 41922, + "alamo": 34632, + "alan": 9563, + "alan": 5773, + "alana": 43405, + "aland": 34304, + "aland": 6819, + "alar": 34333, + "alarm": 11321, + "alarming": 37209, + "alarms": 31236, + "alarts": 31422, + "alas": 7276, + "alas": 22412, + "alaska": 9562, + "alaskan": 33898, + "alastair": 42062, + "alay": 30289, + "alay": 36450, + "alaya": 36397, + "alb": 45248, + "alba": 25254, + "alban": 10882, + "albania": 29170, + "albanian": 47721, + "albans": 44119, + "albany": 17359, + "albat": 42797, + "albeit": 38984, + "alber": 6413, + "albert": 34174, + "albert": 9507, + "alberta": 11048, + "alberto": 22714, + "albi": 18512, + "albino": 48062, + "albion": 24071, + "albu": 2216, + "album": 40712, + "album": 2431, + "albums": 10705, + "albuquerque": 31079, + "alcat": 35361, + "alche": 37909, + "alchemist": 38913, + "alchemy": 39501, + "alco": 6848, + "alco": 45446, + "alcohol": 9426, + "alcoholic": 25098, + "ald": 4539, + "ald": 2928, + "alda": 46440, + "alde": 33114, + "alden": 17155, + "alden": 27710, + "aldenrichards": 20051, + "alder": 18220, + "alder": 46571, + "aldi": 23204, + "aldo": 9933, + "aldridge": 38084, + "alds": 14285, + "aldu": 6505, + "aldub": 10532, + "aldub": 15247, + "ale": 1440, + "ale": 1336, + "alea": 26518, + "aleague": 38909, + "alec": 29804, + "alec": 19954, + "alecoscino": 47948, + "aled": 4970, + "alee": 24515, + "alej": 23440, + "alejandro": 32950, + "alek": 26906, + "alek": 43310, + "aleksand": 48429, + "alem": 11825, + "aleppo": 19258, + "aler": 25674, + "aler": 27335, + "alert": 4662, + "alerts": 22144, + "ales": 44171, + "ales": 5962, + "aless": 21864, + "alessandro": 37344, + "alestine": 31945, + "alex": 2959, + "alex": 4134, + "alexa": 16273, + "alexand": 10696, + "alexander": 25527, + "alexander": 7563, + "alexandra": 19054, + "alexandre": 35711, + "alexandria": 21171, + "alexis": 35023, + "alexis": 14243, + "aley": 21635, + "alf": 27098, + "alfa": 23482, + "alfar": 38870, + "alfie": 28598, + "alfon": 31947, + "alfonso": 41784, + "alfre": 20982, + "alfred": 16553, + "alfredo": 32291, + "algae": 25654, + "algar": 36291, + "algarve": 40290, + "alge": 24336, + "algebra": 33694, + "alger": 18568, + "algeria": 25257, + "algon": 33007, + "algori": 14912, + "algorithm": 23295, + "algorithms": 26039, + "alham": 23352, + "alhamdulil": 35129, + "alhamdulillah": 38982, + "ali": 835, + "ali": 3558, + "alia": 2492, + "aliaa": 36468, + "alian": 3464, + "alias": 40026, + "alibaba": 39231, + "alic": 25265, + "alice": 23759, + "alice": 9192, + "alici": 31630, + "alicia": 20914, + "alie": 8697, + "alien": 22846, + "alien": 9639, + "aliens": 14883, + "alier": 39493, + "alies": 38086, + "alife": 41347, + "alife": 21100, + "alig": 21272, + "alight": 36157, + "align": 31160, + "aligned": 29292, + "alignment": 27267, + "alik": 31141, + "alike": 12665, + "alim": 42075, + "alin": 42746, + "alin": 40063, + "alina": 39529, + "aline": 21799, + "aling": 5169, + "alion": 19049, + "alis": 21308, + "alis": 20114, + "alisa": 38918, + "alisation": 42143, + "alise": 36718, + "alised": 25099, + "alism": 5607, + "alison": 28653, + "alison": 16970, + "alist": 44900, + "alist": 3320, + "alistair": 40551, + "alistic": 22302, + "alists": 5653, + "alit": 45566, + "alities": 27925, + "ality": 1694, + "alive": 40467, + "alive": 4716, + "aliz": 30979, + "alization": 8026, + "alize": 10268, + "alized": 6141, + "alizer": 38922, + "alizes": 26181, + "alizing": 13023, + "alk": 30246, + "alk": 21577, + "alkal": 33450, + "alkaline": 39210, + "all": 813, + "all": 615, + "alla": 13884, + "alla": 14000, + "allabout": 43996, + "allah": 6378, + "allan": 36552, + "allan": 15404, + "allblacks": 47728, + "allday": 35862, + "alle": 4870, + "alle": 29478, + "alled": 7379, + "alleg": 7456, + "allegations": 16992, + "alleged": 12133, + "allegedly": 14177, + "alleges": 45051, + "allegh": 41479, + "allegheny": 47851, + "allegi": 28832, + "allegiance": 30955, + "allen": 16712, + "allen": 6386, + "allenge": 31387, + "aller": 10116, + "aller": 30630, + "allergic": 28809, + "allergies": 28247, + "allergy": 24408, + "allery": 32542, + "alles": 43354, + "allevi": 31682, + "alleviate": 44799, + "alley": 36205, + "alley": 10329, + "allez": 49137, + "alli": 4123, + "alli": 15268, + "alliance": 45404, + "alliance": 8945, + "alliances": 48403, + "allianz": 45740, + "allie": 25040, + "allied": 20045, + "allies": 17277, + "alligator": 28574, + "allin": 45007, + "allin": 22395, + "alline": 48182, + "alling": 2992, + "allis": 45309, + "allison": 34602, + "allison": 16578, + "allman": 42611, + "allo": 8107, + "allo": 18389, + "allocated": 42716, + "allocation": 35139, + "allon": 46693, + "allot": 26363, + "allotment": 33750, + "allow": 5645, + "allow": 6722, + "allowance": 35696, + "allowed": 7885, + "allowing": 12458, + "allows": 9966, + "alloy": 22467, + "alls": 1997, + "allstar": 31247, + "allstar": 22974, + "allstars": 31198, + "allthe": 29253, + "allu": 20157, + "alluarjun": 39333, + "allure": 41814, + "ally": 7461, + "ally": 769, + "alm": 28303, + "alma": 32933, + "alma": 18337, + "alman": 29394, + "almanac": 41268, + "almighty": 21898, + "almond": 15646, + "almonds": 30468, + "almost": 47534, + "almost": 2671, + "aln": 47203, + "alo": 3435, + "alo": 6183, + "aloe": 30728, + "alog": 15813, + "alogue": 9101, + "aloha": 23160, + "aloils": 49002, + "alom": 22236, + "alon": 14097, + "alon": 42846, + "alone": 4702, + "along": 8300, + "along": 2528, + "alongside": 8646, + "alonso": 25704, + "aloo": 46187, + "alore": 14323, + "alot": 16945, + "alou": 43180, + "aloud": 30028, + "alove": 46669, + "alove": 37045, + "alp": 32020, + "alp": 39342, + "alpac": 30128, + "alpaca": 42561, + "alph": 6720, + "alpha": 11807, + "alpha": 8624, + "alphabe": 45796, + "alphabet": 22335, + "alphon": 37865, + "alpine": 17055, + "alps": 18191, + "already": 2426, + "alright": 10866, + "als": 23982, + "als": 938, + "alsace": 49388, + "also": 1446, + "alt": 9995, + "alt": 10006, + "alta": 24470, + "alta": 25378, + "altaf": 47342, + "altam": 45624, + "altar": 16385, + "alter": 4949, + "alter": 21393, + "altered": 25201, + "altern": 47463, + "alternate": 15926, + "alternati": 16699, + "alternative": 37327, + "alternative": 8248, + "alternatives": 25041, + "alth": 23463, + "alth": 5863, + "although": 9421, + "alti": 35531, + "alties": 17276, + "altitude": 23241, + "altman": 48100, + "alto": 35053, + "alto": 17518, + "altogether": 45689, + "alton": 41331, + "alton": 36550, + "altrin": 38458, + "altrincham": 44718, + "alty": 5546, + "alu": 4776, + "alu": 27991, + "alum": 5404, + "alum": 10553, + "alumin": 14563, + "alumini": 22908, + "aluminium": 23631, + "aluminum": 15251, + "alumna": 30313, + "alumni": 6646, + "alumnus": 23633, + "alums": 30155, + "alv": 20928, + "alvar": 25196, + "alvarez": 26924, + "alvaro": 41941, + "alves": 38547, + "alvin": 27023, + "alway": 14046, + "alway": 43764, + "always": 24997, + "always": 1466, + "alwx": 32768, + "aly": 6468, + "aly": 12910, + "alyn": 49150, + "alyss": 29490, + "alyssa": 18898, + "alz": 12936, + "alz": 41128, + "alzheim": 15212, + "alzheimer": 21151, + "alzheimers": 34592, + "am": 548, + "am": 687, + "ama": 18206, + "ama": 1696, + "amad": 45095, + "amade": 37366, + "amag": 32049, + "amal": 15315, + "amal": 36753, + "aman": 19890, + "aman": 10110, + "amand": 14560, + "amanda": 10036, + "amar": 6424, + "amar": 19607, + "amara": 48522, + "amari": 42565, + "amarillo": 40449, + "amarine": 45591, + "amarketing": 30788, + "amas": 22716, + "amas": 15667, + "amat": 38664, + "amat": 25455, + "amate": 12453, + "amateur": 14287, + "amaya": 47210, + "amaz": 1185, + "amaze": 24846, + "amazed": 18944, + "amazing": 15949, + "amazing": 1370, + "amazingly": 20368, + "amazon": 13630, + "amazon": 4140, + "amb": 9042, + "amb": 16853, + "amba": 27003, + "ambani": 45967, + "ambas": 5634, + "ambassad": 5758, + "ambassador": 6795, + "ambassadors": 16832, + "ambed": 42089, + "ambedkar": 48131, + "amber": 18292, + "amber": 9986, + "ambi": 11844, + "ambient": 23447, + "ambigu": 35702, + "ambition": 20673, + "ambitions": 34152, + "ambitious": 18666, + "ambro": 17585, + "ambrose": 24253, + "ambu": 34423, + "ambul": 13944, + "ambulance": 15555, + "ambush": 40725, + "amc": 24942, + "amc": 16921, + "amd": 20845, + "ame": 3995, + "ame": 780, + "amed": 5660, + "ameen": 24229, + "amel": 31988, + "amel": 10960, + "ameli": 21599, + "amelia": 21433, + "amell": 48198, + "amen": 18716, + "amen": 12335, + "amend": 12425, + "amendment": 15019, + "amendments": 40901, + "amenities": 30096, + "ament": 27528, + "amer": 17081, + "amer": 16147, + "ameri": 40422, + "americ": 1283, + "america": 2224, + "americafirst": 43216, + "american": 8746, + "american": 2151, + "americana": 26221, + "americanair": 42538, + "americani": 39726, + "americans": 6676, + "americas": 33343, + "americas": 18142, + "ames": 5469, + "ameter": 23393, + "amethy": 30291, + "amethyst": 31485, + "amex": 46390, + "amg": 21324, + "amher": 32311, + "amherst": 39065, + "ami": 6100, + "ami": 3065, + "amic": 25824, + "amic": 21383, + "amid": 18908, + "amid": 11953, + "amide": 30952, + "amidst": 25172, + "amie": 36901, + "amig": 40294, + "amiga": 35329, + "amigo": 44991, + "amigos": 28176, + "amii": 35462, + "amiibo": 38871, + "amily": 36732, + "amin": 14337, + "amin": 20235, + "amina": 47531, + "amination": 30355, + "amine": 35823, + "aming": 3507, + "amino": 33464, + "amir": 26029, + "amir": 21973, + "amis": 29829, + "amish": 24958, + "amit": 15083, + "amit": 25255, + "amitabh": 48124, + "amitshah": 32374, + "aml": 43185, + "amma": 29786, + "amman": 29243, + "ammo": 33474, + "ammunition": 35060, + "amn": 24073, + "amne": 14596, + "amnesia": 41741, + "amnesty": 46330, + "amnesty": 21177, + "amo": 4833, + "amo": 11156, + "amodi": 9826, + "amon": 17492, + "amon": 24046, + "among": 12310, + "among": 4265, + "amongst": 12520, + "amoo": 26977, + "amor": 19977, + "amor": 15973, + "amore": 38937, + "amore": 22691, + "amores": 36338, + "amos": 18133, + "amoto": 25492, + "amount": 6403, + "amounts": 16747, + "amour": 29908, + "amovie": 41062, + "amp": 3521, + "amp": 6259, + "amped": 22640, + "amphi": 16379, + "amphibious": 45206, + "amphitheater": 41285, + "amphitheatre": 44039, + "ample": 34162, + "amples": 14536, + "ampli": 15647, + "amplifier": 31743, + "amplify": 45308, + "amps": 19252, + "ampton": 29410, + "ampton": 9347, + "amr": 30916, + "amreading": 16546, + "amrit": 33849, + "ams": 1396, + "amster": 9110, + "amsterdam": 9441, + "amtrak": 27855, + "amu": 11347, + "amu": 32336, + "amur": 35014, + "amura": 35487, + "amus": 36269, + "amuse": 21421, + "amuse": 44367, + "amused": 30212, + "amusement": 32570, + "amusic": 20266, + "amusing": 31789, + "amwriting": 9660, + "amy": 10547, + "amy": 5187, + "an": 514, + "an": 550, + "ana": 6588, + "ana": 1388, + "anab": 34742, + "anada": 27948, + "anag": 12115, + "anagh": 40774, + "anaheim": 23728, + "anak": 34814, + "anak": 38658, + "anal": 2785, + "analo": 34179, + "analog": 19963, + "analogue": 46031, + "analy": 4611, + "analyse": 47246, + "analyses": 39695, + "analysis": 5296, + "analyst": 14198, + "analysts": 28075, + "analytical": 34550, + "analytics": 8558, + "analyze": 28519, + "analyzing": 32107, + "anam": 29525, + "anan": 37215, + "anand": 25073, + "anand": 22083, + "anap": 41566, + "anarch": 46405, + "anarchi": 39879, + "anarchy": 27707, + "anas": 31382, + "anas": 12633, + "anast": 48902, + "anasta": 22915, + "anastasi": 36534, + "anastasia": 37975, + "anat": 10045, + "anath": 31277, + "anatom": 33759, + "anatomy": 15376, + "anc": 1124, + "anc": 17758, + "anca": 14583, + "ance": 7165, + "ance": 884, + "anced": 5071, + "ancer": 17415, + "ancers": 37296, + "ances": 3515, + "ancestor": 43904, + "ancestors": 24405, + "ancestral": 41615, + "ancestry": 30922, + "anch": 9489, + "anche": 34679, + "ancho": 26610, + "anchor": 20030, + "anchor": 13201, + "anchorage": 31950, + "anchored": 45926, + "anchors": 37830, + "anci": 4192, + "ancient": 31495, + "ancient": 5810, + "ancies": 21647, + "ancing": 7797, + "anco": 15459, + "ancy": 16282, + "ancy": 3633, + "and": 672, + "and": 537, + "anda": 2911, + "andalu": 31443, + "andco": 36302, + "ande": 26889, + "ande": 30354, + "ander": 3740, + "ander": 3935, + "anders": 10880, + "andersen": 32661, + "anderson": 26683, + "anderson": 6510, + "andes": 24052, + "andfriends": 36871, + "andhi": 21617, + "andhra": 32452, + "andi": 28870, + "andi": 14354, + "andie": 46318, + "andme": 42831, + "ando": 35950, + "ando": 5986, + "andolan": 48965, + "andon": 36488, + "andor": 45243, + "andover": 44177, + "andr": 22661, + "andra": 46795, + "andra": 21730, + "andre": 2657, + "andre": 9400, + "andrea": 10895, + "andreas": 20444, + "andrei": 42137, + "andres": 25197, + "andretti": 44291, + "andrew": 11717, + "andrew": 4847, + "andrews": 14506, + "andri": 37208, + "andro": 4417, + "andro": 17980, + "android": 24284, + "android": 5191, + "androidgames": 46572, + "andromeda": 42942, + "andré": 35609, + "ands": 32257, + "andthe": 22111, + "andu": 44200, + "andum": 47266, + "andy": 9447, + "andy": 2888, + "ane": 5846, + "ane": 3051, + "anec": 33965, + "anem": 41395, + "anemone": 49019, + "aneous": 48273, + "anes": 15381, + "anese": 48778, + "anesthe": 30622, + "anesthesia": 43353, + "anew": 39084, + "anew": 47341, + "anews": 20919, + "aney": 22387, + "anfield": 26993, + "ang": 883, + "ang": 2704, + "anga": 11641, + "angames": 43178, + "angan": 28264, + "angas": 46180, + "ange": 2960, + "ange": 3039, + "angel": 5029, + "angel": 5130, + "angela": 12354, + "angeles": 7382, + "angeli": 15265, + "angelic": 41038, + "angelica": 38582, + "angelina": 28890, + "angelo": 14342, + "angelou": 41328, + "angels": 7809, + "anger": 32737, + "anger": 6788, + "angerous": 39716, + "angers": 29756, + "angh": 34030, + "angi": 28003, + "angi": 24301, + "angie": 18859, + "angle": 21749, + "angle": 6946, + "angled": 32322, + "angler": 22284, + "anglers": 41608, + "angles": 18627, + "anglesey": 31850, + "anglia": 32076, + "anglic": 28322, + "anglican": 33284, + "angling": 36824, + "anglo": 39515, + "anglo": 30408, + "ango": 19090, + "angola": 36636, + "angor": 41740, + "angp": 19992, + "angry": 33910, + "angry": 9054, + "angs": 18441, + "angst": 41714, + "angu": 11209, + "angular": 43584, + "angular": 24981, + "angularjs": 48608, + "angus": 19688, + "ani": 1326, + "ani": 3624, + "ania": 9866, + "anian": 9945, + "anians": 39393, + "anic": 23113, + "anie": 26697, + "anie": 7671, + "anil": 28589, + "anil": 34619, + "anim": 2190, + "animal": 10697, + "animal": 4668, + "animalrights": 42859, + "animals": 4995, + "animate": 40076, + "animated": 13360, + "animation": 10344, + "animations": 42870, + "animator": 42591, + "anime": 23314, + "anime": 6469, + "anin": 45735, + "aning": 30972, + "anir": 27089, + "anirud": 35278, + "anirudhofficial": 45917, + "anis": 40986, + "anis": 47556, + "anism": 20947, + "anist": 16729, + "anistan": 9727, + "aniston": 47344, + "anit": 23683, + "anita": 18544, + "anium": 14794, + "anj": 22443, + "anja": 43440, + "anjali": 38834, + "anjo": 47353, + "ank": 13339, + "ank": 10029, + "anka": 45324, + "ankara": 34309, + "ankle": 14777, + "ankles": 48688, + "ann": 850, + "ann": 5424, + "anna": 13821, + "anna": 2160, + "annab": 22336, + "annabelle": 47661, + "annah": 39166, + "annah": 14327, + "annak": 41720, + "annan": 32166, + "annapolis": 34491, + "annas": 48467, + "anne": 9139, + "anne": 4083, + "anned": 27352, + "anner": 12642, + "annes": 24343, + "annette": 36821, + "annex": 42958, + "annex": 46389, + "anni": 2438, + "anni": 13728, + "annie": 37270, + "annie": 12173, + "annies": 43184, + "annihil": 32734, + "annis": 24742, + "anniv": 31399, + "anniver": 29671, + "annivers": 42836, + "anniversaire": 30882, + "anniversary": 3048, + "anno": 9901, + "anno": 26871, + "annon": 26385, + "annot": 30411, + "announ": 1806, + "announce": 3682, + "announced": 4103, + "announcement": 6932, + "announcements": 23735, + "announcer": 33626, + "announces": 6500, + "announcing": 11593, + "annoy": 45138, + "annoyed": 29863, + "annoying": 15248, + "annu": 21698, + "annual": 2906, + "annually": 23703, + "anny": 34313, + "anny": 5291, + "ano": 5617, + "ano": 2658, + "anom": 21612, + "anomaly": 46811, + "anon": 47079, + "anon": 13667, + "anonym": 38605, + "anonymous": 15036, + "anoo": 25690, + "anor": 13243, + "anor": 16596, + "anos": 20132, + "another": 29274, + "another": 1380, + "anova": 24116, + "ans": 24586, + "ans": 885, + "ansari": 40748, + "ansel": 40356, + "answ": 3369, + "answe": 14391, + "answer": 4518, + "answered": 14499, + "answering": 18280, + "answers": 8692, + "ant": 1103, + "ant": 773, + "anta": 3023, + "antag": 41745, + "antal": 39355, + "antalya": 47440, + "antan": 32899, + "antarc": 21338, + "antarctic": 27077, + "antarctica": 22587, + "ante": 19311, + "ante": 9769, + "antebellum": 41683, + "antelope": 39177, + "anten": 35517, + "antenna": 26370, + "anter": 46508, + "antes": 14927, + "antgrasso": 39074, + "anth": 3737, + "anth": 29741, + "antha": 47981, + "anthe": 34167, + "anthem": 12504, + "anthi": 45261, + "anthology": 21009, + "anthony": 17477, + "anthony": 6113, + "anthro": 10019, + "anthropo": 18538, + "anthropology": 32407, + "anthus": 37639, + "anti": 3120, + "anti": 3564, + "antibio": 18954, + "antibiotic": 34387, + "antibiotics": 29499, + "antibody": 49018, + "antic": 8260, + "anticip": 11435, + "anticipate": 38280, + "anticipated": 18605, + "anticipating": 48067, + "anticipation": 26983, + "antics": 37126, + "antidote": 45476, + "antifa": 35926, + "antigua": 39910, + "antine": 17641, + "antino": 27818, + "antioxid": 23010, + "antioxidant": 37452, + "antioxidants": 34208, + "antiqu": 21745, + "antique": 46517, + "antique": 9060, + "antiques": 17365, + "antis": 19748, + "antisemitism": 36630, + "antit": 37833, + "antitrust": 49343, + "antlers": 47720, + "antly": 5265, + "anto": 16826, + "anto": 24486, + "antoine": 25188, + "anton": 5497, + "anton": 19644, + "antoni": 39958, + "antonio": 30497, + "antonio": 7842, + "antony": 30707, + "antrim": 40252, + "ants": 1589, + "antv": 47520, + "antw": 44460, + "antwer": 26970, + "antwerp": 33797, + "antz": 25684, + "anu": 8537, + "anu": 17152, + "anup": 29617, + "anus": 27084, + "anush": 22765, + "anushka": 42080, + "anushka": 39822, + "anushkasharma": 44203, + "anwar": 34261, + "anxi": 9021, + "anxiety": 11103, + "anxious": 27793, + "any": 1307, + "any": 1504, + "anya": 11173, + "anybody": 10071, + "anyi": 41632, + "anymore": 7372, + "anyone": 2302, + "anything": 3582, + "anytime": 13924, + "anyway": 8931, + "anyways": 19778, + "anywhere": 8863, + "anz": 14445, + "anz": 19425, + "anza": 14669, + "anzac": 31977, + "ao": 7313, + "ao": 5703, + "aoa": 47119, + "aoc": 31918, + "aofficial": 30840, + "aoki": 33602, + "aol": 40643, + "aon": 30928, + "aon": 48476, + "aor": 32044, + "aos": 46860, + "ap": 688, + "ap": 2728, + "apa": 36954, + "apa": 13537, + "apac": 34320, + "apache": 23921, + "apal": 38017, + "apan": 36562, + "apar": 9161, + "apark": 32528, + "apart": 6474, + "apart": 7803, + "aparthe": 25121, + "apartheid": 26597, + "apartment": 8285, + "apartments": 15791, + "aparty": 26767, + "apat": 31755, + "apathy": 18145, + "apc": 20300, + "apd": 44563, + "ape": 6098, + "ape": 2609, + "apec": 47530, + "aper": 13681, + "aper": 5858, + "apers": 15846, + "apes": 9550, + "apeu": 19040, + "apex": 41935, + "apex": 23712, + "aph": 16341, + "aph": 29491, + "apha": 47104, + "apho": 21758, + "aphra": 44147, + "api": 23342, + "api": 14674, + "apia": 44259, + "apic": 40679, + "aping": 18456, + "apink": 35725, + "apis": 37575, + "apk": 27648, + "apo": 4089, + "apo": 19758, + "apocaly": 13932, + "apocalypse": 17571, + "apocalyptic": 35675, + "apol": 5023, + "apolice": 45663, + "apolis": 9598, + "apollo": 48213, + "apollo": 11554, + "apolo": 31094, + "apolog": 25530, + "apologe": 42908, + "apologi": 14977, + "apologies": 21959, + "apologise": 39608, + "apologize": 22879, + "apologizes": 35298, + "apology": 20768, + "apor": 21871, + "apore": 6679, + "apost": 20309, + "apostle": 33051, + "apostles": 48457, + "app": 882, + "app": 2231, + "appa": 4884, + "appa": 13110, + "appalach": 30523, + "appalachian": 36806, + "appalling": 44797, + "appar": 26698, + "apparatus": 37716, + "apparel": 13972, + "apparent": 23963, + "apparently": 5287, + "appe": 3748, + "appe": 45949, + "appeal": 9625, + "appealing": 25909, + "appeals": 22447, + "appear": 5544, + "appear": 9308, + "appearance": 7238, + "appearances": 17214, + "appeared": 11561, + "appearing": 18759, + "appears": 8743, + "appell": 43833, + "appen": 37201, + "appen": 26589, + "apper": 18780, + "appet": 21686, + "appeti": 24179, + "appetite": 24481, + "appetizer": 36065, + "applau": 24713, + "applaud": 42152, + "applause": 22650, + "apple": 8629, + "apple": 3055, + "applemusic": 21390, + "apples": 14032, + "appleton": 45250, + "appli": 15495, + "appliance": 33677, + "appliances": 22134, + "applic": 4235, + "applicable": 37927, + "applicants": 28035, + "application": 7241, + "applications": 7341, + "applied": 12636, + "applies": 24910, + "apply": 4356, + "applying": 17965, + "appo": 5433, + "appoint": 36190, + "appointed": 11087, + "appointment": 10890, + "appointments": 23439, + "appoints": 25132, + "apprais": 36972, + "appraisal": 46108, + "appreci": 3474, + "appreciate": 6263, + "appreciated": 9264, + "appreciates": 36573, + "appreciating": 39352, + "appreciation": 9212, + "appreciationday": 37438, + "appreciative": 45074, + "appren": 10582, + "apprentic": 15662, + "apprentice": 19122, + "apprentice": 17985, + "apprentices": 38252, + "apprenticeship": 26939, + "apprenticeships": 35425, + "appro": 2398, + "approach": 7781, + "approach": 6241, + "approached": 36499, + "approaches": 14962, + "approaching": 12164, + "appropri": 8446, + "appropriate": 10768, + "appropriately": 30383, + "appropriation": 49110, + "approval": 13549, + "approve": 19064, + "approved": 9412, + "approves": 18107, + "approx": 18266, + "approxim": 14201, + "approximately": 16128, + "apps": 7020, + "appstore": 31377, + "appt": 48112, + "appy": 34420, + "apr": 39396, + "apr": 11177, + "apra": 37027, + "apric": 25923, + "apricot": 30815, + "april": 23548, + "april": 2484, + "apro": 42712, + "apro": 49051, + "apron": 29502, + "aps": 8868, + "apse": 31843, + "apt": 17921, + "aptly": 47313, + "apu": 22166, + "apur": 36900, + "apur": 45193, + "aq": 14018, + "aq": 26862, + "aqu": 4458, + "aqua": 18613, + "aquaculture": 41885, + "aquaman": 35098, + "aquari": 37605, + "aquarium": 16814, + "aquarius": 38879, + "aquatic": 22658, + "aque": 35927, + "aque": 37268, + "aqui": 36826, + "aquino": 33796, + "ar": 516, + "ar": 625, + "ara": 24161, + "ara": 3340, + "arab": 5405, + "arab": 12028, + "arabia": 11746, + "arabian": 24663, + "arabic": 16709, + "arabs": 39155, + "arac": 47620, + "arach": 37689, + "arag": 41502, + "araj": 45142, + "arak": 23416, + "aram": 19223, + "aram": 21473, + "arama": 49066, + "aran": 20839, + "aran": 19641, + "aras": 36399, + "arat": 30856, + "arav": 35836, + "arbit": 20267, + "arbitr": 22702, + "arbitration": 34845, + "arbor": 33516, + "arbor": 24878, + "arboretum": 41719, + "arc": 4997, + "arc": 11592, + "arca": 25189, + "arca": 37612, + "arcade": 13331, + "arcadia": 38372, + "arch": 2458, + "arch": 8557, + "archa": 45619, + "archae": 10121, + "archaeological": 26163, + "archaeologists": 45035, + "archaeology": 14868, + "archan": 33359, + "archbishop": 23994, + "arche": 22474, + "archer": 21824, + "archers": 38407, + "archery": 23935, + "arches": 30771, + "archi": 4479, + "archie": 20557, + "archipel": 39750, + "archipelago": 43025, + "architec": 3359, + "architect": 12192, + "architects": 13290, + "architectural": 15360, + "architecture": 39038, + "architecture": 4920, + "archival": 39249, + "archive": 42257, + "archive": 10548, + "archived": 42379, + "archives": 9411, + "archy": 15643, + "arctic": 29716, + "arctic": 9138, + "ard": 3793, + "ard": 746, + "arden": 44600, + "arden": 27057, + "ardi": 23932, + "ardi": 19837, + "ardo": 35735, + "ardo": 9394, + "ards": 1654, + "ardu": 20906, + "arduino": 25398, + "are": 1076, + "are": 631, + "area": 2445, + "areas": 5429, + "arec": 18136, + "areclipse": 36030, + "ared": 5369, + "arel": 12798, + "arella": 24784, + "arelli": 48619, + "aren": 4033, + "aren": 4318, + "arena": 5463, + "arenas": 47860, + "arent": 37487, + "arer": 14857, + "arers": 33159, + "ares": 12224, + "arest": 11708, + "aret": 22247, + "areth": 47725, + "aretha": 42090, + "areyou": 37607, + "arez": 13108, + "arg": 27285, + "argent": 7812, + "argentina": 9789, + "argentine": 32582, + "argon": 40737, + "argos": 37443, + "argu": 7440, + "arguably": 30899, + "argue": 19788, + "argued": 48153, + "argues": 30045, + "arguing": 26549, + "argument": 16224, + "arguments": 24693, + "argus": 44300, + "argy": 21066, + "argyle": 36179, + "argyll": 40667, + "ari": 1221, + "ari": 3681, + "aria": 8883, + "arial": 42431, + "arian": 29980, + "arian": 6953, + "ariana": 14892, + "arianag": 23025, + "arianagrande": 23321, + "arianism": 44351, + "arians": 19104, + "arias": 22567, + "arie": 18774, + "ariel": 47959, + "ariel": 21025, + "aries": 5213, + "arif": 46621, + "arily": 12993, + "arin": 29564, + "arin": 18612, + "arina": 29271, + "arine": 29586, + "aring": 2142, + "ario": 8862, + "arios": 25392, + "aris": 15227, + "arise": 26490, + "arist": 12110, + "aristo": 25666, + "aristotle": 49156, + "arities": 31069, + "arity": 16608, + "arium": 11809, + "arius": 21482, + "ariz": 6516, + "arized": 40167, + "arizon": 28936, + "arizona": 7106, + "arjun": 24565, + "arjun": 20477, + "arjuna": 43835, + "ark": 11921, + "ark": 12010, + "arkansas": 12227, + "arkham": 36381, + "arl": 48542, + "arlington": 44940, + "arlington": 17865, + "arly": 3637, + "arm": 5671, + "arm": 4793, + "arma": 15887, + "arma": 38716, + "armad": 37897, + "armada": 34938, + "armagh": 44313, + "armani": 31314, + "armb": 37096, + "armchair": 45757, + "armed": 40471, + "armed": 8202, + "armen": 13145, + "armenia": 22008, + "armenian": 24891, + "armies": 46686, + "armin": 45481, + "arming": 19766, + "armist": 38150, + "armistice": 46765, + "armor": 16167, + "armored": 28214, + "armory": 38610, + "armour": 18503, + "armoured": 42514, + "arms": 5706, + "armstrong": 15005, + "army": 13541, + "army": 3133, + "armys": 27311, + "arn": 9348, + "arn": 37597, + "arnau": 45556, + "arne": 43509, + "arney": 35962, + "arnold": 49096, + "arnold": 13609, + "arns": 46692, + "aro": 7514, + "aro": 11551, + "aroa": 48209, + "arom": 16831, + "aroma": 40143, + "aroma": 26390, + "aromas": 47439, + "aromatherapy": 42584, + "aromatic": 39669, + "aron": 30855, + "aron": 28926, + "aroo": 47581, + "arora": 31897, + "arosa": 44264, + "arose": 44262, + "around": 35615, + "around": 1630, + "arqu": 35654, + "arquitec": 41703, + "arr": 39106, + "arr": 42489, + "arra": 32918, + "arra": 43827, + "arrahman": 44554, + "arran": 45722, + "arrang": 16711, + "arrange": 15410, + "arrange": 26311, + "arranged": 22451, + "arrangement": 23822, + "arrangements": 23792, + "arranging": 35321, + "array": 17293, + "arre": 4374, + "arrell": 28846, + "arrest": 9320, + "arrested": 5845, + "arresting": 43930, + "arrests": 20683, + "arri": 2115, + "arrival": 9073, + "arrivals": 19583, + "arrive": 8851, + "arrived": 3514, + "arrives": 9905, + "arriving": 10884, + "arro": 15729, + "arrog": 26997, + "arrogance": 47025, + "arrogant": 40582, + "arrow": 30920, + "arrow": 11149, + "arrowhead": 46393, + "arrows": 24768, + "arroyo": 45237, + "ars": 42815, + "ars": 864, + "arse": 22665, + "arsen": 5330, + "arsenal": 45234, + "arsenal": 6084, + "arsene": 32117, + "arson": 29937, + "art": 1486, + "art": 794, + "arta": 12031, + "arte": 13482, + "arte": 12947, + "artem": 40387, + "artemis": 45256, + "arten": 37043, + "arter": 29449, + "artery": 40062, + "artes": 48629, + "artforsale": 48239, + "artgallery": 31982, + "arth": 7146, + "arth": 20265, + "arthistory": 39313, + "arthr": 20807, + "arthritis": 22916, + "arthro": 43255, + "arthur": 35660, + "arthur": 8550, + "arti": 1635, + "arti": 34601, + "artic": 3003, + "articho": 30937, + "artichoke": 39647, + "article": 3550, + "articles": 11939, + "articul": 40343, + "articulate": 45444, + "artif": 8950, + "artifact": 37718, + "artifacts": 30249, + "artificial": 19357, + "artificial": 12040, + "artificialintelligence": 20799, + "artillery": 24465, + "artin": 33168, + "artin": 48540, + "artis": 41794, + "artisan": 36389, + "artisan": 21535, + "artisans": 40140, + "artist": 14326, + "artist": 2456, + "artiste": 41402, + "artistic": 12421, + "artiston": 48443, + "artistry": 38570, + "artists": 4899, + "artistson": 32127, + "artistsontwitter": 39469, + "artlovers": 35617, + "arto": 28464, + "artof": 31751, + "artoftheday": 43990, + "arton": 46744, + "arts": 22040, + "arts": 3812, + "artsy": 31588, + "arturo": 38591, + "artwit": 36713, + "artwork": 4188, + "artworks": 26215, + "arty": 45417, + "arty": 25916, + "aru": 13757, + "aru": 23907, + "aruba": 40131, + "arugula": 40770, + "arum": 48732, + "arun": 16105, + "arun": 31877, + "arunach": 47260, + "arunjaitley": 44874, + "arus": 22644, + "arvin": 16971, + "arvind": 21209, + "arvind": 41079, + "arvindkejriwal": 22971, + "arvo": 45726, + "arwx": 29824, + "ary": 4617, + "ary": 856, + "arya": 23594, + "aryan": 34966, + "as": 587, + "as": 601, + "asa": 39676, + "asa": 11914, + "asad": 42376, + "asaki": 22455, + "asam": 40603, + "asan": 22379, + "asan": 17841, + "asana": 42363, + "asant": 25536, + "asants": 37766, + "asap": 24199, + "asap": 10822, + "asar": 24733, + "asar": 49299, + "asb": 31186, + "asbe": 32113, + "asbestos": 33765, + "asc": 22720, + "asc": 23305, + "ascen": 20767, + "ascension": 35499, + "ascent": 36625, + "asci": 12753, + "asco": 25578, + "asco": 17488, + "ascot": 23723, + "ascri": 15506, + "asd": 36988, + "asda": 29391, + "asdf": 36857, + "asdfghj": 42758, + "asdfghjkl": 47660, + "ase": 8083, + "ase": 894, + "asean": 24472, + "aseball": 46903, + "ased": 2134, + "asen": 41085, + "aser": 39615, + "aser": 7209, + "ases": 3762, + "asf": 25863, + "asg": 34813, + "ash": 2067, + "ash": 2612, + "asha": 40572, + "asha": 13472, + "ashamed": 20633, + "ashby": 46531, + "ashe": 48523, + "ashe": 31752, + "asher": 37585, + "ashes": 12587, + "asheville": 28897, + "ashford": 37796, + "ashi": 15563, + "ashi": 15934, + "ashish": 33145, + "ashland": 39938, + "ashleigh": 49356, + "ashley": 17825, + "ashley": 8957, + "asho": 20273, + "ashok": 38141, + "ashore": 31194, + "ashram": 43445, + "ashton": 43264, + "ashton": 12228, + "ashtra": 18118, + "asi": 3596, + "asi": 12562, + "asia": 5741, + "asian": 21737, + "asian": 7128, + "asiangames": 49108, + "asians": 36771, + "asics": 31097, + "aside": 13676, + "asif": 37302, + "asim": 46050, + "asin": 48432, + "asin": 44347, + "asing": 4194, + "asingly": 15803, + "asion": 31753, + "asis": 12398, + "ask": 11027, + "ask": 2765, + "asked": 3993, + "asking": 5914, + "asks": 7953, + "asl": 41650, + "asleep": 10749, + "asley": 28206, + "asli": 44290, + "asm": 13851, + "asma": 38497, + "asmsg": 19839, + "aso": 30343, + "aso": 27932, + "asober": 43749, + "asocial": 48557, + "ason": 1163, + "asone": 31249, + "asons": 4249, + "asos": 37924, + "asot": 47968, + "asp": 17814, + "asp": 36666, + "asparag": 20301, + "asparagus": 20604, + "aspe": 10894, + "aspect": 19681, + "aspects": 18203, + "aspen": 35695, + "aspen": 25712, + "asper": 32991, + "asph": 28019, + "asphalt": 30574, + "aspir": 12669, + "aspirations": 36127, + "aspire": 24836, + "aspiring": 21862, + "asports": 43695, + "asr": 48052, + "asroma": 41000, + "ass": 12664, + "ass": 5301, + "assa": 47715, + "assad": 18699, + "assam": 19930, + "assan": 26352, + "assange": 27565, + "assas": 9603, + "assassin": 14366, + "assassin": 20029, + "assassinated": 40488, + "assassination": 24907, + "assassins": 34918, + "assassinscre": 36428, + "assassinscreed": 46082, + "assau": 7908, + "assaul": 19596, + "assault": 9679, + "assaulted": 30785, + "assaulting": 44143, + "asse": 3166, + "asse": 38600, + "assel": 37582, + "assemb": 5531, + "assemble": 26169, + "assembled": 22627, + "assemblies": 47406, + "assembling": 38670, + "assembly": 34542, + "assembly": 7059, + "assen": 38651, + "asser": 25665, + "asses": 21596, + "assess": 9209, + "assess": 23211, + "assessed": 44160, + "assessing": 31364, + "assessment": 10590, + "assessments": 32753, + "asset": 48463, + "asset": 13039, + "assets": 13170, + "assi": 2907, + "assi": 39540, + "assie": 31624, + "assign": 14190, + "assigned": 25767, + "assignment": 17342, + "assignments": 34257, + "assim": 36394, + "assimil": 43467, + "assist": 26558, + "assist": 10286, + "assistance": 11685, + "assistant": 6799, + "assistants": 31054, + "assisted": 18095, + "assisting": 24243, + "assists": 12675, + "assn": 44208, + "asso": 17617, + "assoc": 18891, + "associ": 3566, + "associate": 11777, + "associated": 11164, + "associates": 17358, + "association": 5578, + "associations": 33209, + "assor": 38604, + "assorted": 36701, + "assortment": 43112, + "asst": 24767, + "assu": 8328, + "assume": 19294, + "assumed": 37661, + "assuming": 29422, + "assump": 41182, + "assumption": 40773, + "assumptions": 45948, + "assurance": 28408, + "assure": 39161, + "assured": 25591, + "assures": 41988, + "assy": 29940, + "assy": 12963, + "ast": 1761, + "ast": 1242, + "asta": 43269, + "aste": 25033, + "aste": 25579, + "aster": 11013, + "aster": 9526, + "asteroid": 32253, + "asters": 33139, + "asth": 16684, + "asthma": 24610, + "asthour": 41238, + "astic": 15876, + "asting": 29984, + "astle": 46141, + "asto": 47275, + "aston": 24760, + "aston": 13879, + "astoni": 21962, + "astonishing": 27110, + "astonmartin": 40760, + "astor": 26391, + "astor": 47086, + "astoria": 34798, + "astounding": 37748, + "astr": 37609, + "astra": 47205, + "astra": 36079, + "astral": 45889, + "astri": 31243, + "astrid": 46499, + "astro": 8563, + "astro": 15318, + "astrology": 28526, + "astron": 7982, + "astronaut": 18376, + "astronauts": 29733, + "astronom": 23264, + "astronomer": 40036, + "astronomers": 44268, + "astronomical": 39775, + "astronomy": 17472, + "astrophotography": 38559, + "astros": 17598, + "asts": 10452, + "astu": 43137, + "astur": 45795, + "asu": 13157, + "asu": 16001, + "asun": 36044, + "asure": 3813, + "asus": 27269, + "aswell": 42978, + "asx": 38906, + "asy": 8524, + "asy": 2333, + "asylum": 15638, + "asym": 32539, + "at": 527, + "at": 536, + "ata": 4236, + "atable": 23909, + "atal": 24877, + "atal": 24797, + "atan": 33446, + "atar": 20128, + "atar": 7995, + "atari": 21549, + "atas": 30057, + "atay": 39518, + "atc": 28383, + "atch": 15938, + "atd": 33890, + "ate": 992, + "ate": 671, + "ateam": 42784, + "ateau": 16359, + "atec": 37352, + "atech": 31306, + "ated": 14589, + "ated": 943, + "atedly": 24698, + "atee": 32839, + "ateful": 5419, + "atelier": 29932, + "ately": 3862, + "atem": 17116, + "aten": 47984, + "atene": 30405, + "ateneo": 33904, + "ater": 18597, + "ater": 5877, + "ateral": 18819, + "aters": 22364, + "ates": 20370, + "ates": 1150, + "atest": 1705, + "ateur": 43677, + "atf": 28013, + "ath": 1374, + "ath": 1649, + "atha": 22530, + "atham": 23383, + "athan": 41260, + "athan": 26701, + "athe": 8963, + "athed": 47402, + "atheism": 25823, + "atheist": 22571, + "atheists": 47155, + "athen": 29112, + "athena": 30705, + "athens": 13524, + "ather": 6171, + "ather": 1817, + "athered": 34091, + "athers": 17266, + "athi": 28918, + "athing": 36069, + "athle": 3310, + "athlete": 7388, + "athletes": 7125, + "athletic": 33182, + "athletic": 9028, + "athletics": 7019, + "athlon": 14670, + "athome": 38217, + "athon": 4951, + "aths": 28835, + "athy": 34488, + "athy": 13183, + "ati": 591, + "ati": 6751, + "atia": 10908, + "atic": 20248, + "atic": 2647, + "atically": 13558, + "atics": 15666, + "atie": 30137, + "aties": 40060, + "atif": 41592, + "atiku": 37912, + "atile": 15474, + "atility": 23373, + "atime": 20158, + "atin": 36903, + "atin": 23047, + "atine": 39741, + "ating": 25653, + "ating": 1074, + "atio": 35401, + "ation": 2265, + "ation": 656, + "ational": 14205, + "ational": 3108, + "ationals": 44593, + "ationday": 20082, + "ations": 986, + "atis": 45456, + "atis": 41142, + "atism": 45638, + "ative": 18422, + "ative": 1648, + "atively": 11929, + "atives": 5629, + "ativity": 25166, + "atkins": 27734, + "atkinson": 28908, + "atl": 5411, + "atl": 10629, + "atla": 36043, + "atlan": 6818, + "atlanta": 39964, + "atlanta": 6839, + "atlantic": 28804, + "atlantic": 8189, + "atlantis": 27790, + "atlas": 15775, + "atle": 21170, + "atleast": 33231, + "atleti": 46067, + "atletico": 27501, + "atm": 14127, + "atmo": 8271, + "atmosphere": 10506, + "atmospheric": 24223, + "ato": 7987, + "ato": 4364, + "atoday": 26799, + "atom": 22418, + "atom": 24031, + "atomic": 18996, + "atoms": 41434, + "aton": 31525, + "aton": 10012, + "atop": 17455, + "ator": 10748, + "ator": 1962, + "atore": 28314, + "atorial": 32040, + "atories": 35678, + "atorium": 41306, + "ators": 3389, + "atory": 5920, + "atos": 41643, + "atour": 42967, + "atown": 24000, + "atp": 38105, + "atp": 19817, + "atr": 43247, + "atra": 20227, + "atra": 14401, + "atravel": 36981, + "atre": 46057, + "atri": 13882, + "atri": 38889, + "atric": 32238, + "atric": 13652, + "atrics": 36253, + "atrist": 41879, + "atrium": 29725, + "atrix": 43003, + "atro": 18724, + "atroc": 36197, + "atrocities": 37551, + "atry": 28334, + "ats": 46890, + "ats": 1032, + "atsu": 26531, + "att": 1017, + "att": 7103, + "atta": 7282, + "atta": 9146, + "attach": 43676, + "attach": 35653, + "attached": 11038, + "attachment": 28638, + "attack": 24971, + "attack": 3815, + "attacked": 12366, + "attacker": 39288, + "attackers": 47701, + "attacking": 16813, + "attacks": 7321, + "attain": 46459, + "attar": 37110, + "attemp": 4933, + "attempt": 7409, + "attempted": 17408, + "attempting": 18195, + "attempts": 15610, + "atten": 4084, + "atten": 32408, + "attenborough": 45860, + "attend": 9841, + "attend": 5802, + "attendance": 11928, + "attendant": 35424, + "attended": 8140, + "attendees": 14648, + "attending": 6696, + "attends": 22248, + "attention": 4936, + "atters": 30675, + "atthe": 21489, + "atti": 49265, + "atti": 16235, + "attic": 26766, + "attire": 21222, + "attitude": 10648, + "attitudes": 27611, + "attle": 14685, + "attle": 5030, + "attn": 25677, + "attor": 8856, + "attorney": 10372, + "attorneys": 29113, + "attrac": 7154, + "attract": 17010, + "attracted": 28493, + "attracting": 31909, + "attraction": 16807, + "attractions": 22307, + "attractive": 12231, + "attracts": 31024, + "attribu": 24624, + "attributed": 37520, + "attributes": 40763, + "attu": 43173, + "atty": 36705, + "atu": 15191, + "atu": 24295, + "atuesday": 34841, + "atul": 1744, + "atul": 43948, + "atum": 48295, + "atur": 14986, + "aturday": 29027, + "ature": 25305, + "ature": 4490, + "atures": 7358, + "atus": 14795, + "atv": 19598, + "atwood": 45680, + "atwork": 39680, + "atx": 34849, + "atx": 20136, + "aty": 40974, + "aty": 33107, + "atz": 30432, + "au": 627, + "au": 2566, + "aua": 45906, + "aub": 45938, + "auberg": 49382, + "aubre": 25899, + "aubrey": 34110, + "auburn": 42269, + "auburn": 14534, + "auc": 24489, + "auch": 43024, + "auck": 14588, + "auckland": 16072, + "auction": 48160, + "auction": 6462, + "auctioned": 41073, + "auctions": 24876, + "aucus": 47374, + "aud": 16107, + "aud": 19711, + "audi": 5091, + "audi": 10277, + "audible": 33227, + "audience": 6863, + "audiences": 22328, + "audio": 13792, + "audio": 5766, + "audiobook": 26282, + "audit": 12505, + "audit": 17625, + "auditi": 37377, + "audition": 18673, + "auditions": 21134, + "auditor": 38050, + "auditorium": 15063, + "audre": 16075, + "audrey": 18812, + "audu": 27934, + "audubon": 40275, + "auer": 33460, + "auf": 28924, + "aug": 15397, + "aug": 5720, + "auga": 22797, + "augh": 28310, + "augh": 14005, + "augmente": 48356, + "augmented": 32708, + "augu": 2610, + "august": 24353, + "august": 3171, + "augusta": 26144, + "augustine": 27397, + "augustus": 36835, + "auk": 19058, + "aul": 20695, + "aul": 34391, + "ault": 47253, + "ault": 10219, + "aun": 10608, + "aun": 38721, + "aunt": 12685, + "auntie": 23783, + "aunty": 29528, + "aur": 8156, + "aur": 17282, + "aura": 27728, + "aure": 36010, + "aureli": 35980, + "auror": 30067, + "aurora": 13500, + "aus": 10624, + "aus": 7630, + "ausa": 37384, + "ausbiz": 46543, + "ausch": 33926, + "auschwitz": 36523, + "ausopen": 27831, + "ausp": 35039, + "auspicious": 38806, + "auspol": 8241, + "aussi": 19762, + "aussie": 40230, + "aussie": 14424, + "aussies": 35727, + "aust": 26301, + "aust": 25418, + "austen": 29885, + "auster": 25030, + "austerity": 26982, + "austin": 12845, + "austin": 5125, + "austinmahone": 34678, + "austr": 2518, + "australi": 13798, + "australia": 3444, + "australian": 23630, + "australian": 6258, + "australians": 31488, + "austri": 8946, + "austria": 11960, + "austrian": 20638, + "ausv": 35206, + "ausvotes": 34661, + "aut": 12343, + "auth": 2381, + "auth": 38247, + "authent": 18158, + "authentic": 41266, + "authentic": 10369, + "authentication": 39746, + "authenticity": 35734, + "autho": 34552, + "author": 14447, + "author": 4358, + "authored": 37928, + "authori": 19207, + "authorities": 12729, + "authority": 10524, + "authorization": 48854, + "authorized": 28463, + "authors": 10765, + "auti": 8200, + "autism": 36256, + "autism": 11244, + "autisma": 43324, + "autistic": 29360, + "auto": 3917, + "auto": 5668, + "autobiography": 31509, + "autodesk": 40415, + "autograph": 10657, + "autograph": 13722, + "autographed": 16309, + "autographs": 17376, + "autoimmune": 45509, + "autom": 4114, + "automate": 43203, + "automated": 19022, + "automatic": 12126, + "automatically": 20725, + "automation": 12328, + "automobi": 44813, + "automobile": 25258, + "automotive": 12607, + "auton": 13100, + "autonews": 43975, + "autonom": 17870, + "autonomous": 20722, + "autonomy": 39223, + "autopsy": 44436, + "autos": 31118, + "autoshow": 46788, + "auts": 21140, + "autu": 5445, + "autum": 31783, + "autumn": 28940, + "autumn": 6110, + "autumnal": 35481, + "aux": 18154, + "aux": 8909, + "auxiliary": 37778, + "av": 722, + "av": 8484, + "ava": 12385, + "avage": 31505, + "avail": 1651, + "avail": 16686, + "availability": 17551, + "available": 1685, + "aval": 18012, + "avalan": 23970, + "avalanche": 25815, + "avalley": 45082, + "avalon": 30436, + "avan": 27971, + "avan": 33351, + "avant": 24305, + "avar": 33423, + "avatar": 18219, + "ave": 10062, + "ave": 4860, + "avec": 25828, + "aved": 47918, + "avel": 46817, + "avel": 48088, + "aven": 5963, + "aven": 32971, + "aveng": 21935, + "avenger": 24799, + "avengers": 39413, + "avengers": 12016, + "avengersendgame": 49342, + "avent": 22700, + "avenue": 7042, + "aver": 8788, + "aver": 11403, + "average": 6254, + "averaged": 37310, + "averages": 48982, + "averaging": 35266, + "avery": 20313, + "aves": 14023, + "avfc": 21304, + "avg": 19452, + "avgeek": 11114, + "avi": 3324, + "avi": 11297, + "avia": 38710, + "avian": 24115, + "aviation": 27717, + "aviation": 7617, + "aviator": 38921, + "aviators": 48011, + "avici": 46192, + "avicii": 49158, + "avid": 19118, + "avier": 14598, + "avila": 45339, + "aville": 40689, + "avin": 46204, + "avis": 45163, + "avis": 19765, + "aviv": 22130, + "aviva": 47122, + "aviz": 27607, + "avl": 44749, + "avo": 4496, + "avo": 32400, + "avoc": 12291, + "avocado": 14135, + "avocados": 48911, + "avoi": 16797, + "avoid": 30448, + "avoid": 5983, + "avoidance": 47983, + "avoided": 32103, + "avoiding": 22086, + "avoids": 48220, + "avon": 22790, + "avon": 17348, + "avril": 37763, + "avs": 31896, + "avut": 44472, + "avy": 29973, + "aw": 808, + "aw": 5557, + "awa": 4820, + "awa": 6872, + "await": 20769, + "awaited": 20092, + "awaiting": 14872, + "awaits": 15635, + "awak": 9776, + "awak": 41387, + "awake": 14695, + "awaken": 35412, + "awakening": 17017, + "awakens": 23191, + "awal": 42447, + "awal": 35090, + "awan": 48869, + "awan": 20420, + "awar": 5745, + "award": 36310, + "award": 2047, + "awarded": 7368, + "awarding": 37089, + "awards": 34528, + "awards": 2320, + "aware": 4427, + "aware": 7196, + "awareness": 19217, + "awareness": 4823, + "awarenessmonth": 34278, + "awarenessweek": 35294, + "away": 21088, + "away": 1520, + "aways": 12782, + "awaz": 18586, + "awd": 34846, + "awe": 1693, + "awe": 14106, + "aweather": 42142, + "aweather": 28681, + "awec": 38916, + "aweed": 29724, + "awesom": 16727, + "awesome": 30390, + "awesome": 1848, + "awesomeness": 22430, + "awful": 13617, + "awg": 46350, + "awgs": 35275, + "awh": 39566, + "awhile": 19171, + "awi": 15167, + "awil": 47271, + "awilliams": 42163, + "awk": 8888, + "awk": 40943, + "awkward": 42337, + "awkward": 10304, + "awn": 46222, + "awp": 43300, + "aws": 19658, + "awsome": 47196, + "awson": 36286, + "aww": 11568, + "awww": 15634, + "awwww": 26460, + "awx": 28385, + "ax": 3165, + "ax": 9203, + "axe": 19861, + "axel": 47889, + "axel": 32131, + "axes": 45970, + "axi": 30672, + "axial": 46550, + "axis": 19614, + "axle": 39003, + "axx": 47411, + "ay": 658, + "ay": 551, + "aya": 5917, + "ayala": 39827, + "ayama": 41194, + "ayan": 37781, + "ayan": 16269, + "ayana": 37400, + "ayas": 40904, + "ayat": 44902, + "ayat": 35720, + "aye": 21661, + "aye": 12446, + "ayer": 24852, + "ayers": 42783, + "ayesha": 46570, + "ayi": 33025, + "ayles": 44706, + "ayne": 35669, + "ayo": 21929, + "ayo": 18708, + "ayr": 23002, + "ayr": 36473, + "ayrshire": 32687, + "ays": 785, + "ayu": 40769, + "ayurve": 27185, + "ayurveda": 38986, + "ayush": 44831, + "ayy": 32514, + "ayyy": 41052, + "az": 854, + "az": 5468, + "aza": 22883, + "azad": 37838, + "azalea": 34087, + "azam": 34727, + "azar": 27911, + "azcardinals": 48846, + "aze": 41157, + "aze": 28485, + "azer": 19169, + "azerbai": 20649, + "azerbaijan": 23888, + "azhar": 47019, + "azi": 23914, + "azi": 18452, + "azine": 29140, + "azione": 48335, + "aziz": 41205, + "aziz": 29630, + "azo": 41227, + "azon": 36854, + "azores": 42826, + "azte": 33270, + "aztec": 34749, + "aztecs": 49387, + "azu": 27701, + "azu": 46963, + "azul": 39807, + "azure": 18514, + "azwx": 30262, + "azy": 24783, + "azz": 9817, + "azz": 26453, + "azza": 22255, + "azzi": 18758, + "azzle": 39974, + "azzo": 26779, + "azzur": 37055, + "azzy": 44534, + "añ": 23716, + "años": 41634, + "b": 65, + "b": 321, + "ba": 932, + "ba": 1792, + "baa": 33004, + "baahu": 34145, + "baahubali": 38663, + "bab": 1202, + "bab": 19039, + "baba": 12631, + "babe": 31177, + "babe": 7716, + "babes": 14253, + "babies": 6635, + "babs": 36217, + "babu": 21623, + "baby": 7268, + "baby": 1794, + "babygirl": 39554, + "babylon": 31928, + "babymetal": 45013, + "babys": 22266, + "babysitting": 34186, + "bac": 2791, + "bac": 25867, + "bacca": 40708, + "bach": 11773, + "bach": 8758, + "bachchan": 17690, + "bachel": 11283, + "bachelor": 45508, + "bachelor": 16766, + "bachelore": 26009, + "bachelorette": 29093, + "bacher": 49211, + "back": 1663, + "back": 893, + "backbone": 35635, + "backdrop": 20802, + "backed": 12721, + "backer": 22183, + "backers": 32934, + "background": 5994, + "backgrounds": 28215, + "backing": 14935, + "backlash": 31519, + "backpack": 14894, + "backpacking": 29524, + "backpacks": 37063, + "backs": 7562, + "backseat": 48812, + "backstage": 9236, + "backstreet": 46337, + "backthe": 26127, + "backto": 18703, + "backtoschool": 28730, + "backtothe": 43059, + "backup": 14415, + "backward": 37964, + "backwards": 21283, + "backyard": 12608, + "bacon": 48666, + "bacon": 7104, + "bacter": 11814, + "bacteria": 16556, + "bacterial": 26101, + "bad": 2564, + "bad": 2103, + "bada": 37475, + "badan": 39149, + "badass": 11616, + "baddest": 38112, + "baden": 36690, + "bader": 42254, + "badge": 11301, + "badger": 32686, + "badger": 22363, + "badgers": 22521, + "badges": 20084, + "badlands": 43192, + "badly": 13684, + "badminton": 21412, + "badoo": 33192, + "bados": 25755, + "bae": 32834, + "bae": 6855, + "baek": 18557, + "baek": 32702, + "baekhyun": 21572, + "baes": 46332, + "baf": 13616, + "baff": 35693, + "bafta": 29199, + "bag": 3408, + "bag": 3365, + "bage": 9698, + "bagel": 28777, + "bagels": 37489, + "baggage": 31402, + "bagged": 34047, + "bagh": 21659, + "bagh": 37271, + "baghdad": 30763, + "bago": 25105, + "bags": 6136, + "bagu": 27749, + "baguette": 45334, + "bah": 8372, + "bah": 16685, + "baha": 29592, + "baham": 43718, + "bahamas": 21224, + "bahan": 28704, + "bahn": 33452, + "bahrain": 12503, + "bai": 6232, + "bai": 23339, + "bail": 22933, + "bail": 16986, + "bailey": 27535, + "bailey": 10180, + "bain": 40784, + "bain": 21593, + "bair": 29059, + "baird": 40474, + "bait": 18010, + "baj": 20713, + "baja": 40418, + "baja": 28374, + "bajo": 32619, + "bak": 4059, + "bak": 23742, + "bakar": 41414, + "bake": 20736, + "bake": 11878, + "baked": 10364, + "baker": 27303, + "baker": 7743, + "bakers": 35293, + "bakers": 40231, + "bakersfield": 40149, + "bakery": 13377, + "bakes": 43057, + "bakhta": 44912, + "bakhtawar": 46937, + "bakhtawarbz": 47118, + "baking": 11467, + "baku": 46417, + "baku": 31852, + "bal": 1398, + "bal": 2282, + "bala": 20291, + "balaji": 48694, + "balance": 42894, + "balance": 6827, + "balanced": 15273, + "balances": 37733, + "balancing": 23541, + "balboa": 45098, + "balcony": 16169, + "bald": 11153, + "bald": 14875, + "baldhead": 29191, + "baldwin": 16242, + "bale": 48573, + "bale": 18873, + "bales": 42879, + "bali": 16432, + "bali": 10900, + "balkan": 48499, + "balkans": 42987, + "ball": 3807, + "ball": 1069, + "balla": 42246, + "ballad": 33472, + "ballarat": 46645, + "ballard": 31750, + "baller": 49194, + "baller": 25655, + "ballerina": 34962, + "ballers": 34173, + "ballet": 10703, + "balli": 29406, + "ballin": 47444, + "ballin": 33057, + "balling": 47588, + "ballis": 46675, + "ballistic": 36667, + "ballo": 8871, + "ballon": 36469, + "balloon": 13634, + "balloons": 18130, + "ballot": 14185, + "ballots": 35051, + "ballpark": 26080, + "ballroom": 15493, + "balls": 6927, + "bally": 17275, + "bally": 29451, + "balm": 24962, + "balmain": 45929, + "balo": 12395, + "baloch": 23173, + "balochistan": 21918, + "balot": 44615, + "balotelli": 45721, + "bals": 44154, + "balsam": 29121, + "balsamic": 32654, + "balt": 24441, + "balti": 8400, + "baltic": 23817, + "baltimore": 38502, + "baltimore": 9582, + "balu": 38093, + "bam": 6383, + "bam": 12686, + "bama": 20021, + "bambam": 34538, + "bambi": 46596, + "bamboo": 49322, + "bamboo": 16748, + "ban": 1159, + "ban": 2777, + "bana": 18428, + "banan": 38410, + "banana": 8922, + "bananas": 19121, + "banc": 39252, + "band": 4613, + "band": 1963, + "banda": 31865, + "bandai": 42054, + "bandana": 39265, + "bandcamp": 32229, + "banded": 37804, + "bandic": 44400, + "bandit": 27639, + "bandits": 33940, + "bandra": 41393, + "bands": 7858, + "bandung": 29512, + "bandwagon": 36432, + "bandwidth": 48859, + "bane": 9597, + "banerjee": 48102, + "banff": 29565, + "bang": 3524, + "bang": 6907, + "bangalore": 14697, + "banger": 24872, + "bangers": 38311, + "banging": 33033, + "bangkok": 12351, + "bangla": 10339, + "bangla": 45928, + "bangladesh": 11245, + "bangle": 37634, + "bangor": 31190, + "bangs": 27992, + "bangtan": 39131, + "bani": 19732, + "banjo": 27014, + "bank": 7061, + "bank": 2723, + "banker": 27316, + "bankers": 30599, + "bankholiday": 48868, + "banking": 9566, + "bankno": 49201, + "bankof": 39120, + "bankrup": 21904, + "bankrupt": 23077, + "bankrupt": 37288, + "bankruptcy": 23978, + "banks": 6367, + "banksy": 33350, + "bann": 5304, + "banned": 12012, + "banner": 9185, + "banners": 23145, + "banning": 26246, + "bannon": 29710, + "bano": 42947, + "banquet": 14254, + "bans": 15146, + "bant": 23301, + "bant": 46657, + "banter": 25535, + "bao": 39487, + "bao": 20408, + "bap": 7415, + "bap": 23754, + "bapti": 15477, + "baptism": 36765, + "baptist": 13274, + "baptiste": 45770, + "baptized": 45400, + "bar": 1040, + "bar": 2411, + "bara": 19345, + "barack": 18670, + "barack": 22481, + "barackobama": 18885, + "barak": 47419, + "barak": 16260, + "barang": 38446, + "barb": 24173, + "barb": 20913, + "barbados": 26992, + "barbar": 7906, + "barbara": 10937, + "barbarian": 42530, + "barbe": 18372, + "barbecue": 23501, + "barber": 19517, + "barber": 12296, + "barbershop": 37707, + "barbican": 47668, + "barbie": 16923, + "barca": 22942, + "barcel": 6134, + "barcelon": 47820, + "barcelona": 6412, + "barclay": 48877, + "barclay": 45276, + "barclays": 29538, + "bard": 39812, + "bard": 17514, + "bare": 16023, + "bare": 14318, + "barefoot": 30327, + "barely": 12684, + "bargain": 15076, + "bargaining": 41282, + "bargains": 34126, + "barge": 28272, + "bari": 21428, + "bari": 28016, + "barista": 31078, + "barit": 46300, + "bark": 32333, + "bark": 16560, + "barker": 20618, + "barking": 32676, + "barkley": 30266, + "barley": 22607, + "barlow": 25483, + "barn": 10490, + "barn": 10942, + "barnab": 43272, + "barnard": 44332, + "barne": 42527, + "barnes": 13102, + "barnet": 41943, + "barnett": 27650, + "barney": 24563, + "barns": 43759, + "barnsley": 37109, + "barnsley": 32153, + "baro": 17422, + "baro": 30817, + "baron": 48371, + "baron": 19349, + "baroness": 45056, + "barons": 45596, + "baroque": 25065, + "barr": 39473, + "barr": 22492, + "barra": 28442, + "barra": 33542, + "barrabest": 41376, + "barrac": 40835, + "barracks": 35822, + "barre": 13840, + "barre": 38257, + "barred": 33261, + "barrel": 11703, + "barrels": 22059, + "barren": 46743, + "barrett": 18701, + "barri": 8660, + "barric": 29189, + "barrie": 27090, + "barrier": 15706, + "barriers": 16321, + "barrington": 48954, + "barron": 34881, + "barrow": 42568, + "barrow": 24983, + "barry": 18028, + "barry": 8461, + "barrymore": 49310, + "bars": 8616, + "barstool": 44826, + "bart": 14838, + "bart": 12870, + "bartender": 33498, + "barthol": 48989, + "bartlett": 37130, + "bartol": 38209, + "barton": 48853, + "barton": 20345, + "baru": 16356, + "barun": 38278, + "barunsob": 41398, + "barça": 32788, + "bas": 1244, + "bas": 11420, + "basa": 26142, + "base": 2776, + "base": 4579, + "baseball": 23479, + "baseball": 3470, + "based": 35196, + "based": 2812, + "basel": 42803, + "basel": 20903, + "baseline": 40648, + "baseman": 45910, + "basement": 14792, + "bases": 20496, + "bash": 20462, + "bash": 10972, + "bashing": 37545, + "bashir": 42799, + "basic": 40452, + "basic": 7696, + "basically": 9125, + "basics": 15825, + "basil": 19225, + "basil": 14936, + "basilica": 27879, + "basin": 16117, + "basing": 47321, + "basis": 12278, + "baske": 3713, + "basket": 10338, + "basketball": 40023, + "basketball": 3835, + "baskets": 27787, + "basking": 39769, + "basque": 37175, + "bass": 22831, + "bass": 5992, + "bassett": 45992, + "bassist": 26496, + "bast": 28092, + "basti": 8559, + "bastille": 41874, + "bat": 2121, + "bat": 6575, + "bata": 39277, + "batb": 33962, + "batch": 9413, + "bate": 25034, + "bate": 28277, + "bateman": 41635, + "bates": 21727, + "batgirl": 46460, + "bath": 6064, + "bath": 5713, + "bathing": 20144, + "bathro": 21201, + "bathroom": 8470, + "bathrooms": 26434, + "baths": 19442, + "bathtub": 39942, + "bathurst": 36365, + "bati": 23362, + "bati": 37589, + "batman": 27811, + "batman": 7223, + "baton": 24331, + "bats": 14984, + "batsman": 35432, + "batt": 2407, + "batt": 48595, + "battalion": 20820, + "batter": 12654, + "batter": 31855, + "battered": 34375, + "batteries": 16666, + "battersea": 35839, + "battery": 7870, + "batting": 17401, + "battle": 7344, + "battle": 3528, + "battled": 37837, + "battlefield": 16055, + "battlefront": 42214, + "battleof": 47560, + "battles": 14213, + "battleship": 35165, + "battling": 17268, + "bau": 6055, + "bau": 34840, + "bauer": 22903, + "baugh": 41301, + "baum": 19840, + "bautista": 31881, + "bav": 21075, + "bavaria": 39977, + "bavarian": 44458, + "baw": 19808, + "bax": 21216, + "baxter": 26168, + "bay": 3631, + "bay": 2174, + "baya": 31573, + "bayan": 43895, + "bayarea": 28260, + "bayer": 48548, + "bayer": 29183, + "bayern": 14666, + "baylor": 21721, + "bayou": 33955, + "bays": 40156, + "baz": 10430, + "baz": 25268, + "bazaar": 20070, + "bazar": 49298, + "bb": 1174, + "bb": 3529, + "bba": 27762, + "bball": 15664, + "bbb": 33535, + "bbc": 5123, + "bbc": 5188, + "bbcc": 39052, + "bbce": 33818, + "bbcnews": 29370, + "bbcone": 28259, + "bbcqt": 37343, + "bbcr": 35802, + "bbcra": 17115, + "bbcradi": 49213, + "bbcradio": 22876, + "bbcsport": 49321, + "bbcspringwatch": 37358, + "bbctwo": 40395, + "bbcworld": 47340, + "bbe": 37559, + "bbed": 9077, + "bber": 7933, + "bbers": 36494, + "bbhutto": 28085, + "bbhuttozardari": 28135, + "bbi": 37047, + "bbin": 38553, + "bbing": 9787, + "bbins": 42504, + "bbl": 21961, + "bble": 26570, + "bble": 5924, + "bbled": 37626, + "bbles": 18093, + "bblo": 21231, + "bbloggers": 26614, + "bbly": 43031, + "bbm": 25382, + "bbmas": 22145, + "bbn": 28427, + "bbnaija": 20984, + "bbo": 21892, + "bbq": 41270, + "bbq": 6726, + "bbs": 10002, + "bbuk": 45978, + "bby": 11166, + "bby": 3810, + "bc": 3116, + "bc": 2162, + "bcc": 41509, + "bcci": 36138, + "bce": 36510, + "bcfc": 34359, + "bch": 36684, + "bcn": 25766, + "bcoz": 46373, + "bcpoli": 24389, + "bcs": 24909, + "bcu": 28299, + "bd": 24358, + "bd": 11165, + "bday": 33022, + "bday": 5781, + "bdg": 48418, + "bds": 26732, + "be": 571, + "be": 655, + "bea": 21886, + "bea": 20925, + "beach": 6068, + "beach": 2117, + "beaches": 12183, + "beachlife": 43824, + "beacon": 36883, + "beacon": 18858, + "beacons": 39395, + "bead": 31621, + "bead": 23557, + "beaded": 26661, + "beads": 14099, + "beagle": 30044, + "beak": 36498, + "beal": 45769, + "beale": 39717, + "beam": 35339, + "beam": 13663, + "beams": 23993, + "bean": 16471, + "bean": 5328, + "beanie": 21534, + "beans": 8302, + "bear": 6375, + "bear": 4298, + "bearable": 38608, + "bearcats": 33242, + "beard": 26157, + "beard": 9052, + "bearded": 28459, + "beardown": 43687, + "beards": 33020, + "bearer": 30686, + "bearers": 47986, + "bearing": 18370, + "bearings": 42083, + "bearish": 34829, + "bears": 6182, + "beasley": 43349, + "beast": 20847, + "beast": 6957, + "beastmode": 43076, + "beasts": 21771, + "beat": 3774, + "beat": 3018, + "beaten": 10864, + "beater": 41974, + "beati": 44386, + "beating": 10078, + "beatles": 11961, + "beatport": 31421, + "beatrice": 36922, + "beats": 6289, + "beatthe": 40550, + "beatty": 39903, + "beatz": 33363, + "beau": 1016, + "beau": 14298, + "beaufort": 45423, + "beaumont": 32857, + "beaut": 24559, + "beauti": 1154, + "beauties": 14874, + "beautiful": 13662, + "beautiful": 1215, + "beautifully": 10627, + "beauty": 12881, + "beauty": 2488, + "beav": 23260, + "beaver": 26432, + "beaver": 22874, + "beavers": 34513, + "beavs": 43909, + "bebe": 23331, + "bec": 6899, + "bec": 10773, + "became": 5464, + "because": 32714, + "because": 1631, + "becca": 27088, + "bech": 44055, + "beck": 8256, + "beck": 10396, + "becker": 26918, + "beckett": 27249, + "beckham": 18764, + "becky": 32406, + "becky": 18921, + "become": 2989, + "becomes": 6766, + "becoming": 6208, + "bed": 4152, + "bed": 2722, + "bedding": 31761, + "bedford": 20779, + "bedi": 39181, + "bedro": 18415, + "bedroom": 8411, + "bedrooms": 23996, + "beds": 13914, + "bedside": 47473, + "bedtime": 22115, + "bee": 6097, + "bee": 5028, + "beech": 32733, + "beech": 27596, + "beef": 21703, + "beef": 6529, + "beek": 37915, + "been": 33986, + "been": 1025, + "beep": 33432, + "beer": 8885, + "beer": 2544, + "beers": 10907, + "bees": 36249, + "bees": 9100, + "beet": 12582, + "beet": 28621, + "beethoven": 23656, + "beetle": 16534, + "beetles": 36317, + "beetro": 29251, + "beetroot": 31638, + "beets": 36087, + "before": 20898, + "before": 1348, + "beg": 2219, + "beg": 22401, + "began": 8636, + "begg": 36769, + "begging": 25371, + "begin": 19197, + "begin": 4947, + "beginner": 24351, + "beginners": 21930, + "beginning": 5791, + "beginnings": 22581, + "begins": 4635, + "begs": 43531, + "begun": 10514, + "beh": 21971, + "beh": 41612, + "beha": 5737, + "behalf": 11470, + "behave": 28825, + "behaved": 41617, + "behavi": 6149, + "behaving": 40745, + "behavior": 10461, + "behavioral": 25135, + "behaviors": 37741, + "behaviour": 14655, + "behavioural": 46019, + "behe": 42329, + "behin": 2335, + "behind": 2403, + "behindthe": 21104, + "behindthescenes": 26253, + "behold": 15929, + "bei": 38991, + "bei": 23227, + "beige": 26677, + "beij": 11547, + "beijing": 11796, + "bein": 39117, + "bein": 24168, + "being": 13481, + "being": 1265, + "beings": 17998, + "beingsalmankhan": 19637, + "beir": 20176, + "beirut": 22352, + "beit": 26963, + "bek": 46846, + "bek": 26135, + "bekind": 46691, + "bel": 1308, + "bel": 3543, + "bela": 30555, + "belarus": 30849, + "belated": 20256, + "belfast": 35100, + "belfast": 10015, + "belgi": 7001, + "belgian": 15008, + "belgium": 10239, + "belgrade": 30502, + "beli": 1859, + "beli": 45842, + "belichick": 46132, + "belie": 20854, + "beliebers": 27714, + "belief": 14802, + "beliefs": 20575, + "believ": 4972, + "believe": 15819, + "believe": 2649, + "believed": 13380, + "believein": 24294, + "believeinfilm": 37375, + "believer": 26057, + "believers": 28434, + "believes": 12017, + "believing": 19551, + "belinda": 44415, + "belize": 27990, + "bell": 5417, + "bell": 3718, + "bella": 18282, + "bella": 10418, + "bellamy": 34461, + "bellator": 31985, + "belle": 13587, + "belle": 11496, + "belles": 40678, + "bellevue": 32715, + "belli": 43335, + "bellletstalk": 42695, + "bello": 21954, + "bells": 12811, + "bellum": 35493, + "belly": 25901, + "belly": 10404, + "belmont": 25612, + "belo": 8379, + "belo": 41649, + "belong": 16453, + "belong": 13596, + "belonged": 39893, + "belonging": 28193, + "belongs": 14395, + "beloved": 9363, + "below": 3788, + "bels": 43127, + "belt": 36416, + "belt": 7373, + "belts": 21888, + "belvedere": 48003, + "ben": 1465, + "ben": 3518, + "bena": 46249, + "bench": 17770, + "bench": 8771, + "benches": 36349, + "benchmark": 31775, + "bend": 22100, + "bend": 13332, + "bender": 22551, + "bendigo": 48197, + "bending": 33897, + "bene": 12091, + "bene": 47151, + "beneath": 16850, + "bened": 13216, + "benedic": 24402, + "benedict": 47896, + "benedict": 18027, + "benef": 3260, + "benefici": 38593, + "beneficial": 24660, + "beneficiaries": 42160, + "benefit": 6399, + "benefited": 48266, + "benefiting": 29474, + "benefits": 5465, + "benefitting": 47222, + "benevol": 47060, + "benfica": 33873, + "beng": 6962, + "bengal": 17404, + "bengal": 16374, + "bengali": 33774, + "bengals": 23737, + "bengaluru": 21707, + "benghazi": 25967, + "benin": 40296, + "benitez": 46711, + "benjam": 10550, + "benjamin": 38647, + "benjamin": 12131, + "benji": 43548, + "benn": 39097, + "bennet": 48536, + "bennett": 12186, + "benny": 42369, + "benny": 20595, + "beno": 35268, + "benoit": 44373, + "benson": 19578, + "bent": 9809, + "bent": 18369, + "bentley": 16859, + "benton": 30812, + "benz": 27937, + "benz": 13470, + "ber": 867, + "ber": 1516, + "bera": 32802, + "bere": 17458, + "bered": 9193, + "beren": 33654, + "beret": 41658, + "berg": 12022, + "berg": 3294, + "bergen": 22918, + "berger": 35933, + "berger": 13873, + "bergh": 35120, + "bergman": 42597, + "bergs": 43592, + "berk": 15633, + "berke": 14639, + "berkeley": 46049, + "berkeley": 16667, + "berkshire": 27300, + "berlin": 23532, + "berlin": 5891, + "berman": 21514, + "bermu": 21032, + "bermuda": 24644, + "bern": 9195, + "bern": 18382, + "bernade": 46242, + "bernar": 11962, + "bernard": 14579, + "bernardino": 35328, + "bernardo": 27137, + "bernardo": 28696, + "bernardokath": 29081, + "bernat": 40578, + "berni": 18798, + "bernie": 40093, + "bernie": 10503, + "berniesanders": 23745, + "bernstein": 33936, + "berra": 15089, + "berries": 8319, + "berry": 15334, + "berry": 3488, + "bers": 6408, + "berser": 39037, + "bert": 17340, + "bert": 2358, + "berta": 45187, + "berth": 28317, + "bertie": 47182, + "berto": 34073, + "bertr": 36962, + "bertrand": 41594, + "berts": 30205, + "berty": 35973, + "berwick": 40407, + "bery": 11411, + "bes": 26911, + "bes": 3635, + "beside": 13519, + "besides": 17596, + "bespoke": 15612, + "bess": 43791, + "best": 3419, + "best": 949, + "bestbuy": 29749, + "bestest": 31199, + "bestfan": 23880, + "bestfanarmy": 24590, + "bestfriend": 29832, + "bestfriend": 11856, + "bestfriends": 23555, + "besti": 35210, + "bestie": 17188, + "besties": 27346, + "besto": 28615, + "bestof": 27892, + "bestof": 39533, + "bestseller": 25841, + "bestselling": 28632, + "bet": 1051, + "bet": 4430, + "beta": 43188, + "beta": 9505, + "betes": 10255, + "beth": 9993, + "beth": 4892, + "bethan": 18781, + "bethany": 39130, + "bethany": 27952, + "bethe": 12624, + "bethel": 33410, + "bethesda": 32527, + "bethle": 30760, + "bethlehem": 31827, + "betis": 45590, + "beto": 33721, + "betra": 18436, + "betrayal": 33171, + "betrayed": 35692, + "bets": 17107, + "betsy": 28946, + "bett": 17715, + "bett": 20489, + "betta": 36387, + "bette": 35855, + "better": 10320, + "better": 1539, + "bettertogether": 47392, + "betting": 14319, + "betts": 38637, + "betty": 36175, + "betty": 14350, + "between": 1957, + "beu": 38660, + "bev": 40324, + "bev": 30968, + "bever": 9924, + "beverage": 18694, + "beverages": 28521, + "beverley": 39165, + "beverly": 30906, + "beverly": 16728, + "beverlyhills": 45363, + "beware": 14532, + "bewithyou": 36787, + "bex": 18676, + "bex": 24748, + "bexhill": 49200, + "bey": 3234, + "bey": 6767, + "beyon": 11447, + "beyonce": 16632, + "beyoncé": 19219, + "beyond": 22246, + "beyond": 4432, + "bez": 28592, + "bez": 46764, + "bezos": 45000, + "bf": 19858, + "bf": 7990, + "bfc": 37183, + "bff": 11984, + "bffs": 31462, + "bfi": 34244, + "bg": 16674, + "bg": 11295, + "bgc": 47598, + "bgs": 47963, + "bgt": 40665, + "bh": 9930, + "bh": 13603, + "bha": 6144, + "bha": 33068, + "bhafc": 30779, + "bhagat": 49136, + "bhai": 48370, + "bhai": 20508, + "bhak": 34501, + "bham": 31874, + "bham": 23491, + "bhan": 27356, + "bhand": 48679, + "bhar": 9108, + "bharat": 27454, + "bharat": 17430, + "bharti": 46803, + "bhat": 23784, + "bhatt": 36143, + "bhav": 44950, + "bhi": 28943, + "bhi": 21955, + "bhk": 45070, + "bhm": 38741, + "bho": 19721, + "bhopal": 44573, + "bhp": 29776, + "bhs": 29195, + "bhu": 9172, + "bhuban": 38729, + "bhubanes": 41213, + "bhubaneswar": 45888, + "bhushan": 40884, + "bhutan": 32391, + "bhutto": 30153, + "bi": 717, + "bi": 3035, + "bia": 3841, + "biaf": 26961, + "biafra": 36355, + "bian": 19531, + "bian": 9027, + "bianca": 25854, + "bianchi": 45720, + "bians": 28141, + "bias": 11268, + "biased": 22178, + "bib": 44607, + "bib": 21022, + "bibi": 31182, + "bibl": 20912, + "bible": 26738, + "bible": 7583, + "bibli": 23465, + "biblical": 22841, + "biblio": 49131, + "bic": 5960, + "bic": 10675, + "bice": 35589, + "biceps": 46735, + "bick": 27238, + "bicy": 9247, + "bicycle": 11652, + "bicycles": 31326, + "bid": 21035, + "bid": 5553, + "bidding": 23237, + "bide": 45178, + "biden": 19451, + "bids": 16148, + "bie": 5561, + "bie": 4173, + "bieber": 48725, + "bieber": 7535, + "bien": 19176, + "bien": 25742, + "biennale": 33776, + "biennial": 36609, + "bier": 27226, + "bier": 23508, + "bies": 7867, + "big": 1915, + "big": 1205, + "bigbaldhead": 30325, + "bigbang": 41680, + "bigbang": 23734, + "bigdata": 9440, + "bige": 37762, + "bigfoot": 37095, + "bigg": 15312, + "bigg": 35399, + "biggboss": 27056, + "bigger": 6806, + "biggest": 19483, + "biggest": 3505, + "biggie": 28392, + "biggs": 46507, + "bigh": 18106, + "bighit": 35508, + "bigo": 14278, + "bigolive": 20735, + "bigotry": 37269, + "bigre": 36330, + "bih": 33471, + "bihar": 22849, + "bij": 42478, + "bik": 30306, + "bike": 11686, + "bike": 3701, + "biker": 36100, + "biker": 23449, + "bikers": 29468, + "bikes": 9227, + "bikin": 12638, + "biking": 19157, + "bikini": 14531, + "bil": 3092, + "bil": 20506, + "bilateral": 25599, + "bilbao": 34802, + "bild": 35512, + "bile": 25943, + "bilingual": 29623, + "bilities": 13582, + "bility": 4694, + "bill": 4444, + "bill": 2886, + "billboard": 10856, + "billboards": 34741, + "billed": 37558, + "billi": 7693, + "billie": 23990, + "billing": 31797, + "billings": 43615, + "billion": 14520, + "billion": 5729, + "billionaire": 19475, + "billionaires": 41590, + "billions": 20742, + "bills": 9810, + "billsmafia": 48845, + "billy": 15626, + "billy": 6814, + "bilt": 44770, + "bilt": 26654, + "bim": 46737, + "bim": 24775, + "bin": 4849, + "bin": 5346, + "binance": 43520, + "binary": 23497, + "bind": 44513, + "binder": 30541, + "binding": 21287, + "bine": 34848, + "bing": 24818, + "bing": 5665, + "binge": 22600, + "bingham": 43785, + "bingham": 47296, + "bingo": 18418, + "bino": 29172, + "bino": 24313, + "bins": 26934, + "bint": 43647, + "bio": 2830, + "bio": 5162, + "biode": 43502, + "biodegradable": 47740, + "biodiversity": 17428, + "biof": 45158, + "biographical": 49232, + "biography": 15423, + "biological": 18821, + "biologist": 35149, + "biology": 9796, + "biom": 13010, + "biomar": 44549, + "biomass": 36746, + "biome": 26218, + "biomed": 29280, + "biomedical": 33117, + "bionic": 46201, + "biop": 15009, + "biopic": 27942, + "bios": 48505, + "biotech": 22514, + "biotechnology": 40375, + "biotic": 33773, + "biotics": 41371, + "bious": 31845, + "bipartisan": 32266, + "bipolar": 37097, + "bique": 27809, + "bir": 921, + "bir": 16284, + "birch": 31569, + "birch": 22907, + "bird": 6908, + "bird": 3329, + "birdie": 29612, + "birdies": 45618, + "birding": 15851, + "birdman": 41915, + "birdphotography": 47999, + "birds": 41951, + "birds": 4337, + "birdwatching": 33497, + "birk": 48289, + "birken": 40661, + "birmin": 37482, + "birmingham": 38580, + "birmingham": 7720, + "birth": 1128, + "birth": 5397, + "birthday": 7381, + "birthday": 1166, + "birthdays": 17954, + "birthplace": 31429, + "biryani": 46489, + "bis": 5064, + "bis": 14461, + "biscu": 11532, + "biscuit": 18731, + "biscuits": 18248, + "bisexual": 36829, + "bish": 33690, + "bish": 31461, + "bishop": 20625, + "bishop": 8024, + "bishops": 31579, + "bison": 19741, + "bistro": 21770, + "bit": 3010, + "bit": 2010, + "bitcoin": 30848, + "bitcoin": 6366, + "bite": 41613, + "biting": 23016, + "bits": 7747, + "bitt": 39251, + "bius": 45525, + "bix": 46579, + "biz": 8212, + "biz": 5431, + "biza": 47013, + "bizar": 14886, + "bizarre": 16965, + "bizhour": 39462, + "bizitalk": 34929, + "bj": 4592, + "bj": 18229, + "bjj": 27437, + "bjor": 26525, + "bjp": 37264, + "bjp": 6178, + "bk": 15099, + "bk": 14083, + "bkk": 36433, + "bl": 833, + "bl": 9467, + "bla": 2205, + "bla": 19630, + "blac": 21008, + "black": 2025, + "black": 1449, + "blackand": 12809, + "blackandwhite": 23688, + "blackandwhite": 19506, + "blackandwhitephotography": 27544, + "blackberry": 16470, + "blackbird": 38526, + "blackburn": 23789, + "blackfish": 42193, + "blackfriday": 16445, + "blackgirl": 43591, + "blackhawks": 19203, + "blackhistory": 46982, + "blackhistorymonth": 20135, + "blacklist": 30295, + "blacklivesmatter": 23467, + "blackmail": 47295, + "blackops": 43519, + "blackout": 21733, + "blackpanther": 36592, + "blackpink": 20339, + "blackpool": 21031, + "blacks": 16351, + "blackwell": 42642, + "blad": 36635, + "bladder": 33593, + "blade": 10264, + "blades": 16893, + "blah": 29212, + "blaine": 32457, + "blair": 31824, + "blair": 14749, + "blake": 20229, + "blake": 9579, + "blame": 10695, + "blamed": 32906, + "blames": 27841, + "blaming": 29287, + "blan": 4609, + "blanc": 30936, + "blanc": 13301, + "blanca": 40670, + "blanchard": 40177, + "blanche": 34875, + "blanchett": 49378, + "blanco": 26801, + "bland": 44372, + "bland": 30799, + "blank": 15134, + "blanket": 12878, + "blankets": 24042, + "blanks": 48599, + "blasio": 35553, + "blasphe": 36622, + "blast": 46349, + "blast": 5964, + "blasted": 38976, + "blaster": 36341, + "blasting": 26178, + "blasts": 23067, + "blat": 22048, + "blatant": 41391, + "blatt": 39138, + "blau": 45307, + "blaz": 43413, + "blaze": 15497, + "blazer": 17606, + "blazers": 16984, + "blazing": 25267, + "bldg": 22981, + "ble": 1447, + "ble": 1059, + "bleach": 27034, + "bleak": 40355, + "bled": 12006, + "bleed": 23027, + "bleed": 24791, + "bleedblue": 39160, + "bleeding": 20311, + "bleeds": 47339, + "blen": 25651, + "blend": 10780, + "blended": 25813, + "blender": 25066, + "blending": 34307, + "blends": 28572, + "bler": 31305, + "bler": 11979, + "blers": 26930, + "bles": 5763, + "bless": 9640, + "bless": 5387, + "blessed": 4411, + "blessing": 10729, + "blessings": 11185, + "bleu": 30114, + "blew": 18176, + "bley": 43176, + "bli": 1450, + "bli": 28051, + "blin": 9678, + "blin": 5406, + "blind": 17248, + "blind": 8351, + "blinded": 49149, + "blindness": 38812, + "blinds": 32449, + "bling": 39764, + "bling": 7097, + "blink": 18976, + "bliss": 28531, + "bliss": 12893, + "blissful": 42145, + "blit": 39327, + "blitz": 42151, + "blitz": 17548, + "blizz": 13075, + "blizzard": 16111, + "blk": 42950, + "blk": 22872, + "blm": 30957, + "bln": 47348, + "blo": 1204, + "blo": 25505, + "blob": 49312, + "bloc": 30961, + "block": 4638, + "block": 4593, + "blockade": 33489, + "blockbuster": 19939, + "blockchain": 6653, + "blocked": 9106, + "blocker": 44767, + "blocking": 12652, + "blocks": 10113, + "blog": 16376, + "blog": 2589, + "blogg": 33282, + "blogged": 41380, + "blogger": 21352, + "blogger": 7806, + "bloggerrt": 48898, + "bloggers": 11627, + "blogging": 18090, + "blogpost": 41842, + "blogs": 16682, + "bloke": 24384, + "blom": 48996, + "blon": 7958, + "blond": 32426, + "blonde": 10711, + "blondes": 45130, + "blondie": 39236, + "bloo": 2373, + "blood": 9231, + "blood": 3590, + "blooded": 41946, + "bloodh": 48480, + "bloods": 39539, + "bloody": 38568, + "bloody": 9468, + "bloom": 7311, + "bloom": 10257, + "bloomberg": 43109, + "bloomberg": 21238, + "bloomfield": 40342, + "blooming": 45175, + "blooming": 19266, + "bloomington": 34731, + "blooms": 21439, + "bloss": 10017, + "blossom": 14472, + "blossoms": 21916, + "blot": 41710, + "blou": 44506, + "blouse": 23525, + "blow": 15230, + "blow": 10211, + "blower": 25832, + "blowing": 12087, + "blown": 11848, + "blowout": 34857, + "blows": 21063, + "blr": 47250, + "bls": 39458, + "blu": 1263, + "blu": 10273, + "blue": 3829, + "blue": 1746, + "bluebells": 47150, + "blueberries": 29551, + "blueberry": 18251, + "bluebird": 40747, + "bluec": 43194, + "bluef": 41174, + "bluegrass": 26241, + "bluejays": 18684, + "blueprint": 30594, + "blues": 17566, + "blues": 5159, + "blueslyrix": 47068, + "bluet": 13469, + "bluetooth": 14052, + "bluewave": 40025, + "bluff": 27232, + "bluffs": 48844, + "blum": 34818, + "blumen": 38714, + "blun": 34472, + "blunt": 19305, + "blur": 12102, + "blur": 27976, + "bluray": 36818, + "blurred": 38013, + "blurry": 21977, + "blush": 22889, + "blvd": 12578, + "bly": 20930, + "bly": 4426, + "bm": 4773, + "bm": 15916, + "bma": 42573, + "bmc": 27807, + "bmi": 40642, + "bmo": 39083, + "bms": 34074, + "bmw": 26637, + "bmw": 7869, + "bmx": 22535, + "bn": 10496, + "bn": 7992, + "bnb": 20010, + "bnha": 49336, + "bnp": 47910, + "bnw": 35903, + "bo": 647, + "bo": 2525, + "boa": 14732, + "boar": 7837, + "boar": 35473, + "board": 10419, + "board": 1972, + "boarded": 43052, + "boarder": 37414, + "boardgame": 47829, + "boardgames": 32646, + "boarding": 10086, + "boardroom": 47937, + "boards": 7963, + "boardwalk": 29043, + "boast": 44467, + "boasts": 30309, + "boat": 12426, + "boat": 4440, + "boath": 45461, + "boating": 21951, + "boats": 10080, + "boatsales": 46244, + "bob": 8444, + "bob": 4423, + "boba": 39948, + "bobb": 16891, + "bobble": 38796, + "bobblehead": 33451, + "bobby": 17847, + "bobby": 7816, + "bobc": 26153, + "bobcat": 37896, + "bobcats": 27568, + "bobo": 38939, + "bobs": 45533, + "boc": 27307, + "boc": 39042, + "boca": 26094, + "bock": 24961, + "bod": 17904, + "bod": 26340, + "boda": 42030, + "bode": 28452, + "bode": 40429, + "bodega": 47350, + "bodied": 36892, + "bodies": 9799, + "bodily": 49119, + "body": 7132, + "body": 1774, + "bodybuilding": 24538, + "bodyguard": 35565, + "boe": 23476, + "boe": 21773, + "boeh": 38002, + "boehner": 44599, + "boeing": 48135, + "boeing": 11857, + "boer": 44889, + "boer": 40768, + "bog": 23426, + "bog": 28318, + "bogo": 35769, + "bogota": 47059, + "bogus": 42907, + "boh": 43238, + "bohe": 40541, + "bohemi": 21552, + "bohemian": 25753, + "boho": 25444, + "boi": 37129, + "boi": 12673, + "boil": 31332, + "boiled": 23886, + "boiler": 28212, + "boiler": 25615, + "boiling": 32019, + "bois": 47742, + "bois": 21640, + "boise": 23304, + "bok": 26671, + "bok": 15289, + "boko": 30929, + "boks": 40216, + "bol": 2860, + "bol": 8413, + "bola": 12840, + "bold": 26975, + "bold": 8911, + "boldand": 48413, + "boldly": 44778, + "boli": 12722, + "bolic": 27343, + "bolivia": 28628, + "bollah": 36336, + "bolly": 25302, + "bollywood": 32448, + "bollywood": 9604, + "bolo": 40236, + "bolog": 22818, + "bologna": 27513, + "bolster": 47304, + "bolt": 13131, + "bolton": 48757, + "bolton": 16598, + "bolts": 26028, + "bom": 3012, + "bom": 19469, + "bomb": 18091, + "bomb": 6331, + "bombar": 25544, + "bombardier": 42700, + "bombay": 48602, + "bombay": 23890, + "bombed": 24542, + "bomber": 15436, + "bombers": 21786, + "bombing": 14475, + "bombings": 43236, + "bombs": 14410, + "bombshell": 36340, + "bon": 1871, + "bon": 4216, + "bona": 33342, + "bonanza": 40304, + "bond": 37022, + "bond": 6826, + "bonded": 37390, + "bondi": 40092, + "bonding": 19609, + "bonds": 15786, + "bone": 22502, + "bone": 6195, + "bones": 9476, + "bonfire": 23151, + "bongo": 47519, + "boni": 32269, + "boni": 46356, + "bonita": 42896, + "bonjour": 33176, + "bonkers": 39865, + "bonn": 38969, + "bonnar": 47191, + "bonnaroo": 48777, + "bonne": 25844, + "bonnet": 30636, + "bonnie": 18555, + "bono": 24476, + "bons": 42883, + "bonsai": 44129, + "bonus": 8164, + "bonuses": 35144, + "boo": 824, + "boo": 7317, + "boogie": 22639, + "book": 2828, + "book": 1116, + "bookboost": 31257, + "bookclub": 34438, + "bookday": 26327, + "booked": 12584, + "booker": 21302, + "bookfest": 39381, + "booking": 10145, + "bookings": 18345, + "booklet": 27405, + "bookmark": 33596, + "bookof": 45629, + "bookreview": 27362, + "books": 44382, + "books": 2161, + "bookshelf": 34821, + "bookshop": 24705, + "bookstore": 17999, + "bookstores": 46416, + "bookworm": 20743, + "boom": 9609, + "boom": 7121, + "boomer": 33819, + "boomer": 31766, + "boomers": 37988, + "booming": 33487, + "boon": 24979, + "boon": 35821, + "boone": 23453, + "boop": 45047, + "boost": 44639, + "boost": 6260, + "boosted": 37631, + "booster": 20877, + "boosters": 46859, + "boosting": 28480, + "boosts": 29247, + "boot": 10843, + "boot": 8087, + "bootcamp": 22051, + "booted": 42564, + "booth": 47895, + "booth": 3971, + "booths": 32653, + "booties": 46188, + "bootleg": 38139, + "boots": 7319, + "booze": 24341, + "bop": 19720, + "bor": 1141, + "bor": 15093, + "bora": 24736, + "bord": 36891, + "bordeaux": 22009, + "border": 16304, + "border": 6177, + "borderlands": 38676, + "borders": 13900, + "bore": 14084, + "bore": 24638, + "bored": 8933, + "boredom": 31460, + "boretum": 38902, + "borg": 14770, + "borgh": 17180, + "boring": 12519, + "boris": 31212, + "boris": 15704, + "borisjohnson": 44481, + "born": 17695, + "born": 2683, + "borne": 42910, + "borne": 9328, + "borneo": 33332, + "bornon": 41811, + "bornonthisday": 42757, + "boro": 26796, + "boro": 7974, + "borough": 22761, + "borough": 6203, + "borrow": 22293, + "borrowed": 28224, + "borrowing": 41045, + "borussia": 36764, + "bos": 14885, + "bos": 9644, + "bosa": 46946, + "bosch": 42009, + "bosch": 19466, + "bosco": 36960, + "bose": 23142, + "bosh": 42244, + "bosni": 42924, + "bosnia": 31396, + "boss": 17935, + "boss": 4206, + "bosses": 23906, + "boston": 11540, + "boston": 4399, + "bostonmarathon": 44533, + "bot": 4136, + "bot": 6947, + "botan": 12554, + "botanic": 32560, + "botanical": 21026, + "botany": 22612, + "botd": 34451, + "both": 36575, + "both": 2212, + "bother": 21125, + "bothered": 27997, + "botox": 43449, + "bots": 13721, + "botswana": 27584, + "bott": 3520, + "bott": 37225, + "bottle": 37306, + "bottle": 5392, + "bottled": 29331, + "bottlen": 46439, + "bottles": 9754, + "bottling": 42006, + "bottom": 32314, + "bottom": 5931, + "bottoms": 31524, + "bou": 3728, + "bou": 23165, + "bouchard": 47930, + "boudo": 48827, + "bought": 4142, + "boul": 24830, + "boulder": 18260, + "boule": 17652, + "boulevard": 19504, + "boun": 5993, + "bounce": 14316, + "bouncing": 32060, + "bouncy": 43415, + "bound": 15140, + "bound": 4567, + "boundaries": 18690, + "boundary": 21344, + "bounds": 37469, + "bounty": 21142, + "bouquet": 20961, + "bour": 2934, + "bour": 35486, + "bourbon": 48118, + "bourbon": 14652, + "bourdain": 48095, + "bourg": 20690, + "bourgeo": 45672, + "bourn": 39143, + "bourne": 13789, + "bourne": 5192, + "bournemouth": 20911, + "bout": 19982, + "bout": 8123, + "bouti": 10926, + "boutique": 12179, + "bow": 2297, + "bow": 4040, + "bowden": 48538, + "bowed": 49130, + "bowel": 36880, + "bowen": 25368, + "bower": 40414, + "bowers": 42238, + "bowie": 13036, + "bowing": 46398, + "bowl": 26719, + "bowl": 3814, + "bowled": 39987, + "bowler": 25528, + "bowlers": 42632, + "bowles": 41611, + "bowling": 10390, + "bowls": 17787, + "bowman": 22052, + "bows": 17000, + "bowser": 38234, + "bowski": 48311, + "box": 2774, + "box": 2063, + "boxed": 24190, + "boxer": 40394, + "boxer": 15363, + "boxers": 31019, + "boxes": 8350, + "boxing": 33669, + "boxing": 5554, + "boy": 2927, + "boy": 1876, + "boyband": 31568, + "boyce": 44480, + "boycot": 46208, + "boycott": 31615, + "boycott": 19559, + "boyd": 18295, + "boyfriend": 7328, + "boyfriends": 36541, + "boyle": 22802, + "boys": 25223, + "boys": 2034, + "boyz": 16152, + "bp": 23410, + "bp": 11558, + "bpa": 43855, + "bpd": 48587, + "bpl": 28901, + "bpm": 40338, + "bps": 37794, + "br": 711, + "br": 7532, + "bra": 1195, + "bra": 5860, + "brac": 6663, + "brace": 8376, + "brace": 9183, + "bracelet": 8969, + "bracelets": 20027, + "braces": 19249, + "brack": 25676, + "bracket": 14780, + "brackets": 36183, + "brad": 4848, + "brad": 9405, + "bradbury": 45097, + "braden": 46842, + "bradford": 15062, + "bradley": 31905, + "bradley": 10952, + "brador": 24062, + "bradshaw": 37556, + "brady": 42494, + "brady": 11117, + "brae": 42874, + "brae": 40040, + "brag": 30110, + "bragg": 38545, + "bragging": 38199, + "brah": 20276, + "brahms": 45114, + "brai": 25048, + "braid": 31067, + "braided": 39997, + "braids": 34221, + "brain": 9454, + "brain": 4812, + "brains": 17129, + "brainstorming": 36607, + "braised": 28363, + "brake": 14937, + "brakes": 23456, + "bral": 31309, + "bram": 14815, + "bram": 39456, + "brampton": 35124, + "bran": 3684, + "bran": 28348, + "brance": 36072, + "brance": 15413, + "branch": 7998, + "branches": 15843, + "brand": 3910, + "brand": 2896, + "branded": 18097, + "brandi": 41003, + "branding": 10841, + "brando": 41892, + "brandon": 20423, + "brandon": 9166, + "brands": 8681, + "brandt": 22552, + "brandy": 26232, + "brane": 32340, + "branson": 28280, + "brant": 28951, + "brant": 47592, + "braries": 46377, + "brary": 24520, + "bras": 22611, + "brasil": 18991, + "brass": 24348, + "brass": 11655, + "brat": 26717, + "brat": 26631, + "brate": 41864, + "braun": 39129, + "braun": 29309, + "brave": 25461, + "brave": 7769, + "braved": 47663, + "bravely": 42303, + "bravery": 25831, + "braves": 14422, + "braving": 43258, + "bravo": 38613, + "bravo": 13006, + "braw": 37871, + "brawl": 26066, + "braxton": 37451, + "bray": 26256, + "bray": 22993, + "braz": 4625, + "brazil": 47459, + "brazil": 6305, + "brazili": 45697, + "brazilian": 12111, + "brb": 25316, + "brc": 40393, + "bre": 887, + "bre": 7782, + "brea": 7318, + "brea": 46538, + "breach": 21363, + "breaches": 45173, + "bread": 18886, + "bread": 5066, + "breads": 43064, + "break": 2206, + "break": 2568, + "breakable": 30691, + "breakaway": 42732, + "breakdown": 14519, + "breaker": 14814, + "breakers": 22270, + "breakfa": 45931, + "breakfast": 30210, + "breakfast": 3290, + "breaking": 14698, + "breaking": 2755, + "breakingbad": 38032, + "breakingnews": 23837, + "breakout": 16752, + "breaks": 7263, + "breakthrough": 18802, + "breakup": 38931, + "breast": 12930, + "breast": 9475, + "breastcancer": 40813, + "breastcancer": 30065, + "breastfeeding": 29033, + "breasts": 37637, + "breath": 9508, + "breath": 9576, + "breathe": 11364, + "breathing": 14959, + "breathtaking": 14709, + "brecht": 34622, + "breck": 44598, + "bred": 46929, + "bred": 16008, + "bree": 7892, + "bree": 37138, + "breed": 28030, + "breed": 13791, + "breeders": 37472, + "breeding": 16544, + "breeds": 29021, + "breen": 48013, + "brees": 46721, + "breeze": 13125, + "breezy": 21451, + "breit": 23864, + "breitbart": 37926, + "brek": 35494, + "bremen": 39861, + "bren": 5209, + "brenda": 23786, + "brendan": 35134, + "brendan": 15414, + "brendon": 36756, + "brennan": 22372, + "brenner": 42941, + "brent": 31439, + "brent": 16355, + "brentwood": 33108, + "brero": 47781, + "bres": 32561, + "bret": 38020, + "bret": 32548, + "brethren": 43134, + "breton": 32290, + "brett": 22591, + "brett": 12394, + "brev": 42882, + "brevi": 39475, + "brew": 5048, + "brew": 7253, + "brewco": 33582, + "brewed": 23238, + "brewer": 20756, + "breweries": 35277, + "brewers": 17618, + "brewery": 8850, + "brewing": 8275, + "brewingco": 45155, + "brews": 21663, + "brewster": 40274, + "brex": 22726, + "brexit": 27666, + "brexit": 5801, + "brgy": 35983, + "bri": 1036, + "bri": 18636, + "bria": 35890, + "brian": 9824, + "brian": 4989, + "brianna": 32308, + "briar": 46119, + "bribe": 40042, + "bribery": 41792, + "bric": 27055, + "brice": 40190, + "brick": 13937, + "brick": 9518, + "bricks": 21029, + "brics": 48196, + "brid": 16995, + "bridal": 36875, + "bridal": 14284, + "bride": 18342, + "bride": 8964, + "brides": 18067, + "bridesma": 28356, + "bridesmaid": 43399, + "bridesmaids": 47754, + "bridg": 20623, + "bridge": 8647, + "bridge": 2465, + "bridgeport": 45201, + "bridges": 11811, + "bridget": 27073, + "bridgewater": 38732, + "bridging": 38109, + "brie": 26622, + "brief": 9435, + "brief": 8954, + "briefed": 47326, + "briefing": 12991, + "briefly": 26980, + "briefs": 29557, + "brien": 13504, + "brier": 43995, + "brig": 11081, + "briga": 46448, + "brigade": 16032, + "briggs": 28108, + "brigh": 6710, + "bright": 10383, + "bright": 4852, + "brighten": 18208, + "brightening": 43929, + "brighter": 18507, + "brightest": 26159, + "brightly": 36298, + "brightness": 42280, + "brighton": 28416, + "brighton": 9470, + "brigitte": 44421, + "brill": 27342, + "brill": 28601, + "brilli": 3821, + "brilliance": 28146, + "brilliant": 4106, + "brilliantly": 26803, + "brin": 25620, + "bring": 11596, + "bring": 2430, + "bringback": 28969, + "bringbackour": 45403, + "bringing": 4777, + "brings": 5138, + "brink": 39296, + "brink": 28796, + "brioche": 45818, + "bris": 9385, + "bris": 15783, + "brisban": 30431, + "brisbane": 42932, + "brisbane": 12407, + "brisk": 43646, + "brisket": 31920, + "bristol": 18159, + "bristol": 8010, + "brit": 2318, + "brit": 20066, + "britain": 40802, + "britain": 6272, + "britanni": 31373, + "britannia": 36188, + "brite": 33827, + "briti": 8155, + "british": 8651, + "british": 3504, + "britishmuseum": 41858, + "britney": 37192, + "britney": 21853, + "britneyspears": 42990, + "brits": 21832, + "britt": 10811, + "britt": 25976, + "brittany": 38187, + "brittany": 18818, + "britton": 37422, + "brium": 46079, + "brixton": 30056, + "bro": 927, + "bro": 4410, + "broad": 3491, + "broad": 12623, + "broadband": 21050, + "broadcast": 8967, + "broadcaster": 29005, + "broadcasting": 14403, + "broadcasts": 46742, + "broader": 36029, + "broadway": 34599, + "broadway": 9092, + "broc": 15587, + "broccoli": 19094, + "broch": 21419, + "brochure": 25275, + "brock": 14841, + "brock": 16745, + "brodie": 42150, + "brody": 29608, + "broke": 42165, + "broke": 6509, + "broken": 26126, + "broken": 5107, + "broker": 34032, + "broker": 20449, + "brokerage": 41327, + "brokers": 28271, + "brom": 18972, + "brom": 33296, + "bromance": 35353, + "bromley": 35715, + "bron": 4011, + "bron": 10243, + "bronco": 43488, + "bronco": 34370, + "broncos": 12516, + "bronson": 37042, + "bronte": 48936, + "bronx": 48310, + "bronx": 17183, + "brony": 21084, + "bronze": 8459, + "broo": 5204, + "brooch": 21207, + "brook": 4782, + "brook": 7322, + "brooke": 28576, + "brooke": 12549, + "brookes": 39707, + "brooklyn": 23253, + "brooklyn": 6983, + "brooks": 42779, + "brooks": 9991, + "broom": 32046, + "broom": 28008, + "broome": 49335, + "bros": 7776, + "broth": 29994, + "brotha": 33974, + "brother": 12697, + "brother": 3157, + "brotherhood": 19059, + "brothers": 4548, + "brou": 27874, + "brough": 21033, + "brought": 4222, + "brov": 42881, + "brow": 6547, + "brow": 15895, + "broward": 34719, + "brown": 6315, + "brown": 2866, + "browne": 28440, + "brownie": 23045, + "brownies": 22312, + "browning": 32241, + "browns": 14051, + "brows": 14998, + "browse": 19060, + "browser": 19768, + "browsing": 29318, + "brox": 43539, + "brs": 47485, + "brt": 46936, + "bru": 1698, + "bru": 31028, + "bruce": 21223, + "bruce": 7085, + "bruh": 17575, + "bruins": 14736, + "bruise": 48048, + "bruised": 46502, + "brum": 23862, + "brum": 28078, + "brun": 6870, + "brunch": 9113, + "brune": 29057, + "brunei": 41898, + "brunette": 35528, + "bruno": 14568, + "brunomars": 41156, + "brunswick": 24012, + "brush": 27969, + "brush": 8594, + "brushed": 30298, + "brushes": 21550, + "brushing": 35072, + "brussels": 11020, + "brut": 39499, + "brutal": 42144, + "brutal": 14556, + "brutality": 31348, + "brutally": 28132, + "brute": 47552, + "brux": 49093, + "bry": 6587, + "bry": 28228, + "bryan": 16134, + "bryan": 10412, + "bryant": 12256, + "bryce": 19895, + "bryn": 36569, + "bryn": 42877, + "bryson": 38990, + "bs": 11783, + "bs": 1329, + "bsa": 46619, + "bsb": 23070, + "bsbi": 41728, + "bsbibotany": 42086, + "bsc": 32031, + "bsd": 41848, + "bse": 46341, + "bsf": 48314, + "bsgo": 48474, + "bsp": 47977, + "bst": 19698, + "bsu": 46385, + "bt": 3317, + "bt": 4205, + "btc": 10315, + "btcc": 30759, + "btn": 44681, + "bto": 35516, + "btob": 29379, + "btr": 39767, + "bts": 15154, + "bts": 4007, + "btsarmy": 30302, + "btsbbmas": 35297, + "btsx": 44971, + "btv": 38541, + "btw": 9520, + "btwn": 28284, + "bu": 609, + "bu": 5831, + "bub": 27704, + "bub": 33158, + "bubb": 9739, + "bubba": 28149, + "bubble": 28687, + "bubble": 10799, + "bubblegum": 48078, + "bubbles": 17648, + "bubbly": 31034, + "buc": 8207, + "buccane": 32830, + "buccaneers": 38058, + "buch": 22623, + "bucha": 43582, + "buchan": 27237, + "buchanan": 28975, + "bucharest": 37013, + "buck": 6061, + "buck": 11433, + "bucket": 22596, + "bucket": 10498, + "bucketlist": 30778, + "buckets": 27168, + "buckeye": 34549, + "buckeyes": 30741, + "buckingham": 28736, + "buckle": 21948, + "buckley": 25905, + "bucks": 6103, + "bucky": 35916, + "bucs": 20011, + "bud": 2942, + "bud": 10737, + "buda": 18520, + "buda": 49012, + "budapest": 19202, + "budd": 7296, + "buddha": 13981, + "buddhism": 23744, + "buddhist": 18697, + "buddies": 14543, + "budding": 31992, + "buddy": 40948, + "buddy": 6557, + "budge": 32005, + "budget": 46758, + "budget": 5639, + "budgeting": 43789, + "budgets": 36419, + "buds": 14665, + "budweiser": 40900, + "buen": 15640, + "buena": 30876, + "buenas": 48529, + "bueno": 46202, + "buenos": 26055, + "buf": 44417, + "buff": 5456, + "buff": 21416, + "buffal": 25836, + "buffalo": 31231, + "buffalo": 8054, + "buffalob": 38831, + "buffalobills": 44352, + "buffe": 13724, + "buffer": 33050, + "buffet": 17829, + "buffett": 34081, + "buffs": 28906, + "buffy": 33356, + "bug": 14453, + "bug": 8162, + "bugatti": 35451, + "buggy": 28963, + "bugs": 13850, + "buh": 31406, + "buhari": 14661, + "buick": 22000, + "buil": 1354, + "build": 22739, + "build": 3289, + "builder": 14474, + "builders": 17694, + "building": 21206, + "building": 2307, + "buildings": 8866, + "builds": 16449, + "buildthe": 41497, + "built": 45824, + "built": 3874, + "buk": 28084, + "buk": 24317, + "buka": 47778, + "bukit": 39888, + "bul": 2572, + "bul": 10200, + "bula": 18726, + "bulaga": 41575, + "bular": 32187, + "bulb": 22373, + "bulbs": 24808, + "bulgar": 15424, + "bulgaria": 20295, + "bulgarian": 38693, + "bulge": 47603, + "bulk": 19643, + "bull": 4537, + "bull": 6029, + "bulldo": 37675, + "bulldog": 34828, + "bulldog": 15611, + "bulldogs": 13916, + "bullet": 14340, + "bullet": 12465, + "bulletin": 19638, + "bulletproof": 43212, + "bullets": 22117, + "bullied": 34689, + "bullies": 39050, + "bullion": 49114, + "bullish": 22142, + "bullock": 33198, + "bullpen": 38081, + "bulls": 10313, + "bully": 43111, + "bully": 20190, + "bullying": 13548, + "bum": 27683, + "bum": 14226, + "bumble": 25585, + "bumble": 39303, + "bumblebee": 36911, + "bummed": 48456, + "bump": 9783, + "bump": 15877, + "bumped": 22495, + "bumper": 17881, + "bumping": 40196, + "bumps": 21115, + "bun": 2591, + "bun": 13665, + "bunch": 7796, + "bund": 41905, + "bunde": 18841, + "bundesliga": 21582, + "bundle": 11793, + "bundled": 47228, + "bundles": 29834, + "bundy": 37332, + "bung": 44748, + "bungal": 29549, + "bungalow": 33696, + "bunk": 41236, + "bunker": 23615, + "bunnies": 28998, + "bunny": 34198, + "bunny": 9258, + "buns": 22235, + "bunting": 30695, + "buon": 31350, + "buon": 48498, + "bur": 1039, + "bur": 17362, + "burbank": 34862, + "burberry": 30412, + "burch": 44588, + "burden": 18687, + "bure": 11902, + "bureau": 32098, + "bureau": 15400, + "burg": 19505, + "burg": 3499, + "burge": 20522, + "burger": 22356, + "burger": 6548, + "burgers": 13007, + "burgess": 26211, + "burgh": 18141, + "burgh": 4965, + "burgl": 25554, + "burglar": 43365, + "burglary": 32573, + "burgring": 40823, + "burgundy": 23650, + "buri": 46348, + "buri": 42614, + "burial": 22012, + "buried": 14233, + "burk": 48822, + "burke": 15340, + "burle": 27891, + "burlesque": 33732, + "burlington": 23370, + "burma": 30305, + "burmese": 47906, + "burn": 7934, + "burn": 4285, + "burnaby": 47541, + "burne": 27246, + "burned": 15022, + "burner": 23243, + "burnett": 28558, + "burnham": 36111, + "burning": 46107, + "burning": 8405, + "burnley": 24653, + "burnout": 36078, + "burns": 10234, + "burnt": 15185, + "burr": 30879, + "burrell": 49045, + "burrito": 23473, + "burritos": 47245, + "burroughs": 41337, + "burrows": 44846, + "burst": 13005, + "bursting": 32566, + "bursts": 37026, + "burt": 27162, + "burton": 42354, + "burton": 12704, + "burundi": 33595, + "bury": 12276, + "bury": 3899, + "burys": 32362, + "bus": 1319, + "bus": 2840, + "busan": 40172, + "busc": 35000, + "busch": 20475, + "buses": 12879, + "bush": 11191, + "bush": 6867, + "bushes": 37578, + "busiest": 32764, + "busine": 4598, + "busines": 25364, + "business": 8346, + "business": 1716, + "businesses": 7287, + "businessman": 25635, + "buss": 47764, + "bust": 31299, + "bust": 9959, + "busted": 18643, + "buster": 37219, + "buster": 12094, + "busters": 16362, + "busting": 29622, + "busy": 39332, + "busy": 4354, + "but": 2201, + "but": 767, + "butch": 35102, + "butcher": 18732, + "butchers": 42334, + "bute": 39240, + "butes": 14630, + "butler": 35867, + "butler": 10702, + "butt": 12500, + "butt": 31523, + "butte": 31678, + "butter": 5427, + "butter": 6952, + "butterflies": 16232, + "butterfly": 9738, + "buttermilk": 40180, + "butternut": 36867, + "buttery": 45535, + "button": 45480, + "button": 8007, + "buttons": 16188, + "butts": 25309, + "buu": 42313, + "buuren": 47752, + "buxton": 41370, + "buy": 11632, + "buy": 2131, + "buyer": 14682, + "buyers": 14663, + "buying": 6566, + "buys": 15560, + "buzz": 7866, + "buzz": 8706, + "buzzard": 47434, + "buzzer": 38064, + "buzzfeed": 26613, + "buzzing": 18511, + "bv": 18958, + "bv": 35861, + "bvb": 22454, + "bw": 17672, + "bw": 15120, + "bway": 26652, + "bwfc": 40918, + "bwo": 45902, + "bx": 33633, + "by": 1713, + "by": 638, + "bye": 20076, + "bye": 4460, + "byes": 47958, + "byl": 34994, + "byn": 46917, + "byn": 11890, + "byo": 28039, + "bypass": 26530, + "byr": 15534, + "byrd": 30369, + "byrne": 19676, + "byron": 43504, + "byron": 19775, + "bys": 26740, + "bystand": 46138, + "byte": 42798, + "bytes": 39538, + "bythe": 36621, + "byu": 41072, + "byu": 23770, + "byz": 35406, + "byzantine": 44081, + "bz": 13631, + "bé": 40365, + "bü": 38706, + "c": 66, + "c": 322, + "ca": 772, + "ca": 1684, + "caa": 19316, + "cab": 3033, + "cab": 11912, + "cabaret": 26263, + "cabbage": 18407, + "cabe": 32731, + "cabello": 34371, + "caber": 29062, + "cabernet": 33730, + "cabin": 14178, + "cabine": 23354, + "cabinet": 9937, + "cabinets": 33083, + "cabins": 48455, + "cable": 7925, + "cables": 22408, + "cabo": 37318, + "cabo": 28370, + "cabrera": 42338, + "cabs": 42048, + "cac": 8298, + "cac": 23872, + "cacao": 38022, + "cache": 28993, + "caching": 40655, + "cactus": 19794, + "cad": 6297, + "cad": 20166, + "caday": 34187, + "cadbury": 44698, + "caddy": 41521, + "cade": 10497, + "cade": 17306, + "cadet": 22764, + "cadets": 19160, + "cadillac": 18156, + "cae": 49264, + "caer": 28298, + "caes": 15740, + "caesar": 21642, + "caesars": 42162, + "caf": 3471, + "caf": 20867, + "cafc": 30748, + "cafe": 15201, + "cafe": 4979, + "cafes": 40166, + "cafeteria": 32817, + "caffe": 18258, + "caffe": 45416, + "caffeine": 22487, + "café": 15304, + "cag": 15714, + "cage": 11838, + "cages": 37939, + "cah": 40519, + "cahill": 33185, + "cai": 38971, + "cai": 36116, + "cain": 13747, + "caine": 16799, + "cair": 15804, + "cair": 46659, + "cairn": 31264, + "cairn": 42467, + "cairngor": 44067, + "cairns": 32941, + "cairo": 19615, + "cait": 14116, + "caitlin": 47768, + "caitlin": 26809, + "caitlyn": 35763, + "cajun": 43425, + "cajun": 33044, + "cak": 42986, + "cake": 15295, + "cake": 2972, + "cakeday": 46207, + "cakes": 5950, + "cal": 1198, + "cal": 6372, + "cala": 32133, + "calab": 31795, + "calais": 39886, + "calam": 28841, + "calc": 45055, + "calci": 22824, + "calcium": 27815, + "calcu": 15328, + "calcul": 15734, + "calculate": 37656, + "calculated": 40688, + "calculations": 44605, + "calculator": 26093, + "calculus": 35104, + "calcutta": 42901, + "calder": 29372, + "calder": 36817, + "caldwell": 30484, + "cale": 32674, + "caleb": 19619, + "caled": 28421, + "calend": 6057, + "calendar": 7122, + "calendars": 17229, + "calf": 17508, + "calgary": 27415, + "calgary": 10797, + "calhoun": 38929, + "cali": 2857, + "cali": 16337, + "caliber": 32820, + "calibr": 32597, + "calico": 45379, + "calif": 30839, + "califor": 3526, + "californi": 21303, + "california": 3729, + "call": 7950, + "call": 1620, + "calla": 20658, + "callahan": 43313, + "callaway": 42596, + "callback": 44764, + "calle": 47699, + "calle": 38144, + "called": 2726, + "caller": 30666, + "calli": 16338, + "callie": 36512, + "calligraphy": 27775, + "calling": 4597, + "callister": 49026, + "callme": 42449, + "callof": 41280, + "calls": 4572, + "callum": 23224, + "calm": 34990, + "calm": 7011, + "calming": 30690, + "calorie": 32679, + "calories": 18029, + "cals": 47714, + "calum": 16405, + "calvary": 40169, + "calvert": 47134, + "calves": 31857, + "calvin": 27642, + "calvin": 17345, + "caly": 10244, + "calyp": 29851, + "cam": 1004, + "cam": 5982, + "camar": 31991, + "camber": 44362, + "cambo": 14662, + "cambodia": 17347, + "cambridge": 24651, + "cambridge": 9334, + "cambridgeshire": 46139, + "camden": 38735, + "camden": 17984, + "came": 1986, + "camel": 27005, + "camel": 21914, + "camels": 41357, + "cameo": 19492, + "camer": 4961, + "camera": 3934, + "cameraman": 43347, + "cameras": 12172, + "camero": 20320, + "cameron": 19634, + "cameron": 8057, + "camerondallas": 40587, + "cameroon": 24061, + "camil": 37745, + "camila": 19919, + "camilla": 38897, + "camille": 26741, + "camino": 28529, + "camo": 28702, + "camo": 19716, + "camogie": 39547, + "camou": 23588, + "camoufla": 23667, + "camouflage": 29049, + "camp": 2854, + "camp": 2877, + "campa": 2793, + "campaig": 9448, + "campaign": 44524, + "campaign": 3193, + "campaigner": 46364, + "campaigners": 40272, + "campaigning": 19594, + "campaigns": 15669, + "campan": 31765, + "campbell": 29094, + "campbell": 8806, + "campe": 16672, + "campeon": 49109, + "campeones": 30105, + "camper": 41914, + "camper": 24522, + "campers": 26619, + "campfire": 32530, + "campground": 46969, + "camping": 9982, + "campo": 27600, + "campos": 48077, + "camps": 12806, + "campsite": 44243, + "campu": 19687, + "campus": 4560, + "campuses": 31895, + "camra": 46155, + "camry": 46472, + "cams": 32590, + "can": 950, + "can": 753, + "cana": 28341, + "canad": 13193, + "canada": 2698, + "canadaday": 39800, + "canadi": 4329, + "canadian": 22160, + "canadian": 5255, + "canadians": 18989, + "canadiens": 40932, + "canal": 28585, + "canal": 9535, + "canals": 38483, + "canaria": 47117, + "canary": 40409, + "canary": 24523, + "canberra": 16719, + "canc": 43189, + "cancel": 12026, + "cancel": 21546, + "canceled": 25874, + "cancell": 28027, + "cancellation": 38765, + "cancelled": 13270, + "cancels": 34089, + "cancer": 12690, + "cancer": 3148, + "cancers": 33201, + "cancun": 34721, + "cand": 4986, + "candace": 45623, + "candel": 47834, + "candi": 6034, + "candice": 30024, + "candid": 7884, + "candid": 19206, + "candidacy": 46248, + "candidate": 6475, + "candidates": 8619, + "candied": 43982, + "candies": 46305, + "candle": 18995, + "candle": 12674, + "candlelight": 34724, + "candles": 15472, + "candy": 20741, + "candy": 6417, + "cane": 23644, + "cane": 14716, + "canelo": 43210, + "canes": 21902, + "cani": 35592, + "canine": 27380, + "cann": 4139, + "cann": 23709, + "cannab": 7577, + "cannabis": 31837, + "cannabis": 8861, + "canne": 44252, + "canned": 27290, + "cannes": 13773, + "canni": 26389, + "canning": 38621, + "cannon": 28771, + "cannon": 15661, + "cannons": 46269, + "cannot": 4785, + "canny": 26986, + "cano": 31668, + "cano": 25937, + "canoe": 23503, + "canola": 40389, + "canon": 17749, + "canon": 9310, + "canopy": 26061, + "cans": 13707, + "cant": 13395, + "cant": 5784, + "canteen": 39230, + "canter": 19301, + "canterbury": 22271, + "canti": 42845, + "cantina": 47472, + "canton": 37735, + "canton": 25363, + "cantore": 41769, + "cantwait": 33760, + "canu": 20171, + "canucks": 24321, + "canv": 30714, + "canvas": 22441, + "canvas": 7483, + "canvass": 40054, + "canvassing": 33783, + "cany": 47674, + "canyon": 41246, + "canyon": 9755, + "cao": 29207, + "cap": 1289, + "cap": 3938, + "capabilities": 19512, + "capability": 25885, + "capable": 14742, + "capac": 24665, + "capacity": 8970, + "capcom": 28342, + "cape": 10288, + "cape": 6631, + "capecod": 41339, + "capes": 38785, + "capetown": 20059, + "capit": 6889, + "capita": 41833, + "capital": 11198, + "capital": 5439, + "capitalism": 20068, + "capitalist": 37015, + "capitals": 29579, + "capitol": 43880, + "capitol": 11375, + "capo": 45477, + "capp": 16718, + "capped": 24659, + "capping": 42656, + "cappuccino": 37402, + "capri": 48699, + "capri": 30982, + "capric": 28667, + "capricorn": 46314, + "caps": 23185, + "capsu": 15608, + "capsul": 40341, + "capsule": 20627, + "capsules": 32870, + "capt": 45815, + "capt": 17369, + "captain": 14958, + "captain": 4621, + "captainamerica": 46229, + "captainmarvel": 48492, + "captains": 18706, + "caption": 11327, + "captions": 41878, + "captiv": 19776, + "captivating": 30580, + "captive": 29038, + "captivity": 41141, + "capture": 8818, + "captured": 8020, + "captures": 15305, + "capturing": 19548, + "capu": 44241, + "car": 811, + "car": 1615, + "cara": 20016, + "carab": 32251, + "carac": 30029, + "caracas": 45854, + "caramel": 14788, + "carameli": 41739, + "caramelized": 43854, + "carat": 32981, + "carav": 13814, + "caravan": 18566, + "carb": 21379, + "carbo": 43235, + "carbon": 14038, + "carbon": 7549, + "carbs": 29313, + "carcin": 31587, + "carcinoma": 46810, + "card": 10793, + "card": 2601, + "cardam": 49008, + "cardboard": 19845, + "cardi": 6211, + "cardi": 29677, + "cardiac": 21256, + "cardiff": 22488, + "cardiff": 9781, + "cardigan": 30501, + "cardin": 8457, + "cardinal": 46310, + "cardinal": 16472, + "cardinals": 12837, + "cardio": 15003, + "cardio": 23455, + "cardiology": 37276, + "cardiovascular": 29291, + "cardo": 40625, + "cards": 4094, + "care": 2050, + "care": 1776, + "cared": 27675, + "career": 20609, + "career": 3061, + "careers": 10090, + "careful": 11999, + "carefully": 15789, + "caregi": 22042, + "caregiver": 46372, + "caregivers": 35909, + "careless": 47325, + "carers": 26484, + "cares": 10968, + "caretaker": 48037, + "carey": 14895, + "cargo": 12490, + "cari": 18497, + "cari": 37273, + "carib": 9757, + "caribbean": 10368, + "caribou": 42135, + "caric": 25337, + "caricature": 38857, + "carina": 44357, + "caring": 13083, + "carl": 8273, + "carl": 9482, + "carla": 25552, + "carleton": 46496, + "carlin": 47559, + "carlisle": 23276, + "carlo": 17861, + "carlo": 15266, + "carlos": 9538, + "carlow": 44745, + "carls": 39635, + "carlson": 24114, + "carlton": 18934, + "carly": 23166, + "carly": 22689, + "carlyle": 46555, + "carmel": 30757, + "carmel": 25601, + "carmen": 41427, + "carmen": 18834, + "carmichael": 41657, + "carn": 21597, + "carnage": 31385, + "carnation": 44577, + "carnaval": 47238, + "carne": 17053, + "carne": 42885, + "carnegie": 25287, + "carney": 34194, + "carni": 8438, + "carnival": 36708, + "carnival": 10577, + "caro": 30317, + "caro": 29344, + "carol": 4242, + "carol": 11489, + "carole": 31955, + "carolin": 26418, + "carolina": 7027, + "caroline": 31064, + "caroline": 12641, + "carols": 33269, + "carolyn": 25825, + "carou": 32224, + "carousel": 36665, + "carp": 26085, + "carpen": 15584, + "carpenter": 18475, + "carpet": 6922, + "carpets": 34612, + "carr": 26951, + "carr": 17136, + "carra": 32332, + "carre": 31114, + "carrera": 32952, + "carri": 4739, + "carriage": 47885, + "carriage": 21087, + "carrick": 44052, + "carrie": 30334, + "carrie": 15848, + "carried": 12960, + "carrier": 12308, + "carriers": 26865, + "carries": 17982, + "carrieunderwood": 47338, + "carrington": 48759, + "carroll": 41911, + "carroll": 14893, + "carrot": 15435, + "carrots": 19299, + "carry": 31863, + "carry": 6998, + "carrying": 9920, + "cars": 3346, + "carsforsale": 45222, + "carson": 41766, + "carson": 13171, + "cart": 27705, + "cart": 13065, + "cartag": 45042, + "cartagena": 47157, + "carte": 44949, + "cartel": 30529, + "carter": 27330, + "carter": 7260, + "cartier": 32951, + "carto": 5487, + "carton": 41812, + "cartoon": 33082, + "cartoon": 7651, + "cartoonist": 30793, + "cartoons": 17673, + "cartri": 47084, + "cartridge": 29432, + "cartridges": 49249, + "carts": 27581, + "cartunesapp": 32888, + "caruso": 45192, + "carve": 40152, + "carved": 15127, + "carver": 28850, + "carving": 19428, + "carvings": 48123, + "cary": 22844, + "cas": 1671, + "cas": 13831, + "casa": 14643, + "casablanc": 36572, + "casablanca": 41950, + "casc": 36714, + "casca": 43296, + "cascade": 29065, + "cascades": 46454, + "case": 17698, + "case": 2068, + "cases": 6888, + "casey": 24899, + "casey": 12836, + "cash": 11050, + "cash": 5131, + "cashback": 36368, + "cashe": 32233, + "cashew": 39531, + "cashi": 29517, + "cashier": 34547, + "cashmere": 34566, + "casi": 38350, + "casino": 10473, + "casio": 32261, + "cask": 26299, + "casm": 35198, + "casper": 35892, + "cass": 22556, + "cassandra": 35289, + "casser": 31093, + "casserole": 36045, + "cassette": 19717, + "cassi": 14942, + "cassidy": 21757, + "cassie": 29323, + "cassini": 46554, + "cast": 2509, + "cast": 1970, + "caste": 32693, + "casted": 33838, + "castel": 43306, + "castell": 31792, + "caster": 32101, + "caster": 8449, + "casters": 29721, + "castic": 47737, + "castillo": 30813, + "casting": 7087, + "castle": 12496, + "castle": 3540, + "castles": 24766, + "castro": 16950, + "casts": 10595, + "casu": 15345, + "casual": 10129, + "casually": 18840, + "casualties": 30244, + "casualty": 31222, + "cat": 1481, + "cat": 2368, + "cata": 42279, + "catal": 12792, + "catalan": 30532, + "catalina": 36576, + "catalo": 34740, + "catalog": 20036, + "catalogue": 20985, + "catalonia": 27039, + "catalunya": 44132, + "cataly": 15894, + "catalyst": 25387, + "catan": 45893, + "catap": 39514, + "catar": 35801, + "catastro": 22736, + "catastrophe": 41422, + "catastrophic": 34448, + "catch": 18901, + "catch": 3042, + "catcher": 15965, + "catchers": 39060, + "catches": 17213, + "catching": 8617, + "catchy": 37114, + "catday": 32243, + "cate": 6357, + "cate": 24510, + "cated": 31823, + "categor": 17006, + "categori": 40117, + "categories": 19971, + "category": 9432, + "cater": 16634, + "cater": 38101, + "catering": 16697, + "caterpillar": 27111, + "catfish": 26077, + "cath": 9196, + "cath": 30811, + "cathar": 43784, + "cathe": 7174, + "cathedr": 46370, + "cathedral": 7865, + "catherine": 35035, + "catherine": 12339, + "catho": 7595, + "cathol": 16315, + "catholic": 20382, + "catholic": 7757, + "catholics": 36808, + "cathy": 40326, + "cathy": 22731, + "cation": 21367, + "cato": 33558, + "cats": 38800, + "cats": 3989, + "catsofinstagram": 39901, + "catsoftwitter": 17273, + "catt": 37339, + "cattle": 48799, + "cattle": 13644, + "caturday": 20892, + "catwalk": 36565, + "catwoman": 47251, + "cau": 1121, + "cau": 45529, + "caucus": 18847, + "caught": 4520, + "caul": 23460, + "cauley": 41682, + "caulfield": 44906, + "cauli": 20123, + "cauliflower": 23802, + "cause": 18982, + "cause": 1394, + "caused": 8940, + "causes": 9775, + "causeway": 35034, + "causing": 10779, + "caution": 15656, + "cautious": 36579, + "cav": 4942, + "cav": 45935, + "cava": 48682, + "caval": 24537, + "cavali": 20783, + "cavalier": 44488, + "cavaliers": 30194, + "cavalry": 32467, + "cave": 25441, + "cave": 9654, + "cavendish": 42945, + "caver": 41487, + "caves": 22096, + "cavi": 27360, + "caviar": 31228, + "cavill": 40492, + "cavity": 43156, + "cavs": 16800, + "caw": 38405, + "caw": 43804, + "cawx": 26739, + "cay": 11876, + "cay": 37399, + "cayenne": 43650, + "cayman": 33737, + "caz": 48451, + "cb": 4034, + "cb": 8830, + "cba": 38472, + "cbb": 31487, + "cbc": 14096, + "cbc": 14523, + "cbd": 13176, + "cbe": 43639, + "cbi": 30875, + "cbj": 35608, + "cbn": 26579, + "cbp": 46723, + "cbr": 28762, + "cbs": 16788, + "cbs": 8009, + "cc": 2976, + "cc": 2021, + "cca": 17987, + "ccc": 21856, + "ccd": 48556, + "ccg": 37755, + "cch": 21789, + "cchini": 28467, + "cci": 32942, + "cci": 8196, + "ccl": 43773, + "ccm": 40435, + "cco": 28786, + "ccot": 24950, + "ccp": 43045, + "ccs": 30400, + "cctv": 23097, + "ccu": 49023, + "cd": 4308, + "cd": 4480, + "cda": 45565, + "cdc": 41098, + "cdc": 25779, + "cdn": 8886, + "cdn": 26802, + "cdnpoli": 11645, + "cdo": 47187, + "cdp": 39624, + "cds": 20784, + "cdt": 18455, + "ce": 685, + "ce": 629, + "cea": 28355, + "cean": 34409, + "cean": 37295, + "cease": 32856, + "cease": 25499, + "ceasefire": 38291, + "cebu": 20146, + "cec": 29694, + "cec": 40029, + "cecil": 26987, + "cecil": 27169, + "cecilia": 35440, + "ced": 25634, + "ced": 2323, + "cedar": 24167, + "cedar": 13799, + "cedric": 36608, + "cee": 45966, + "cee": 15015, + "cees": 47914, + "ceil": 27275, + "ceiling": 12374, + "ceilings": 33770, + "cek": 45544, + "cel": 2269, + "cel": 7597, + "cele": 1314, + "celeb": 38862, + "celeb": 19393, + "celebr": 1372, + "celebrate": 31414, + "celebrate": 2694, + "celebrated": 9184, + "celebrates": 7564, + "celebrating": 3382, + "celebration": 4615, + "celebrations": 10825, + "celebratory": 34115, + "celebrities": 17071, + "celebrity": 23981, + "celebrity": 7320, + "celebs": 19803, + "celed": 25741, + "celer": 9621, + "celery": 30990, + "celeste": 29364, + "celesti": 29497, + "celestial": 32669, + "celi": 25567, + "celia": 44489, + "celine": 33644, + "cell": 9316, + "cell": 5533, + "cellar": 24282, + "cellars": 44976, + "cellence": 34687, + "cello": 23013, + "cellphone": 39029, + "cells": 8890, + "cellu": 16791, + "cellular": 23268, + "cels": 24021, + "celsius": 47057, + "celtic": 21897, + "celtic": 10523, + "celticfc": 38612, + "celtics": 16226, + "cem": 41435, + "ceme": 10517, + "cement": 4369, + "cements": 19448, + "cemetery": 11660, + "cen": 1306, + "cen": 30106, + "cena": 21591, + "cence": 24410, + "cency": 41259, + "cene": 30038, + "censor": 24230, + "censor": 44709, + "censored": 30951, + "censorship": 27284, + "census": 23677, + "cent": 1784, + "cent": 3662, + "centenary": 22422, + "centennial": 20895, + "center": 16651, + "center": 2119, + "centered": 24584, + "centers": 14494, + "centi": 48889, + "centime": 48687, + "centr": 2370, + "central": 13448, + "central": 3339, + "centre": 26310, + "centre": 2916, + "centred": 47925, + "centres": 19354, + "centri": 30872, + "centric": 19297, + "centro": 37178, + "cents": 11934, + "centu": 16818, + "centuri": 36816, + "centuries": 19014, + "century": 26134, + "century": 4275, + "ceo": 46340, + "ceo": 3559, + "ceos": 28332, + "cep": 2632, + "cep": 48714, + "ceph": 44343, + "cept": 3678, + "ception": 12346, + "cer": 1364, + "cer": 1925, + "cera": 34608, + "ceram": 10677, + "ceramic": 15112, + "ceramics": 22438, + "cere": 3984, + "cere": 22085, + "cereal": 17581, + "cereals": 48618, + "cerebral": 39073, + "ceremon": 15796, + "ceremonial": 33281, + "ceremonies": 21547, + "ceremony": 5193, + "cern": 44851, + "cers": 13638, + "cert": 27522, + "certain": 8526, + "certain": 7883, + "certainly": 10883, + "certainty": 20054, + "certi": 4888, + "certific": 9443, + "certificate": 11786, + "certificates": 25281, + "certification": 14735, + "certified": 9288, + "cerv": 25738, + "cervical": 35953, + "ces": 28715, + "ces": 1604, + "cesar": 37025, + "cesar": 28603, + "cess": 2314, + "cess": 1554, + "cessna": 36596, + "cest": 27245, + "cester": 15769, + "cester": 12718, + "cet": 14960, + "cett": 46708, + "ceu": 37457, + "cevic": 48369, + "cey": 20971, + "cf": 10189, + "cf": 11171, + "cfa": 34521, + "cfb": 32931, + "cfc": 11577, + "cfd": 46171, + "cfl": 46320, + "cfl": 22332, + "cfo": 26937, + "cfp": 40756, + "cfr": 44033, + "cfs": 32835, + "cg": 27118, + "cg": 14740, + "cgc": 38775, + "cgi": 30520, + "ch": 540, + "ch": 634, + "cha": 1587, + "cha": 4541, + "chab": 26670, + "chad": 13095, + "chad": 12923, + "chae": 9460, + "chaf": 38123, + "chag": 27989, + "chai": 31590, + "chai": 18919, + "chain": 13898, + "chain": 3946, + "chained": 34402, + "chains": 14438, + "chainsaw": 37617, + "chainz": 39687, + "chair": 4728, + "chair": 4269, + "chaired": 31664, + "chairing": 42205, + "chairman": 6901, + "chairperson": 31584, + "chairs": 12033, + "chak": 13702, + "chak": 41713, + "chakra": 38304, + "chakra": 33241, + "chal": 7397, + "chal": 30809, + "chale": 38099, + "chalet": 37907, + "chalk": 31362, + "chalk": 17846, + "chall": 2073, + "challeng": 4138, + "challenge": 29462, + "challenge": 2836, + "challenged": 17380, + "challenger": 18228, + "challengers": 46404, + "challenges": 6280, + "challenging": 11754, + "chalmers": 47955, + "cham": 1290, + "cham": 19951, + "chamber": 18983, + "chamber": 7642, + "chamberlain": 32756, + "chambers": 16501, + "chamele": 34759, + "chameleon": 41317, + "champ": 36813, + "champ": 6602, + "champag": 10283, + "champagne": 11007, + "champi": 1680, + "champion": 2643, + "champion": 3950, + "champions": 4227, + "championship": 3429, + "championships": 7047, + "championsleague": 27638, + "champs": 6240, + "chan": 1255, + "chan": 6704, + "chana": 48752, + "chanc": 13931, + "chance": 32940, + "chance": 2594, + "chancellor": 15886, + "chances": 10870, + "chand": 7126, + "chand": 41508, + "chandelier": 30570, + "chandi": 12482, + "chandigarh": 34106, + "chandler": 17595, + "chandra": 27082, + "chandra": 25348, + "chanel": 16951, + "chang": 2233, + "chang": 16461, + "change": 11608, + "change": 1799, + "changeable": 41335, + "changed": 4907, + "changer": 18406, + "changers": 35185, + "changes": 4938, + "changing": 40384, + "changing": 5621, + "changmin": 47410, + "chann": 8804, + "channel": 25837, + "channel": 3847, + "channeling": 28197, + "channels": 13961, + "channing": 37417, + "chant": 18165, + "chant": 13521, + "chanting": 32111, + "chants": 22723, + "chanyeol": 18805, + "chao": 31815, + "chaos": 10853, + "chaotic": 33501, + "chap": 3825, + "chap": 21939, + "chapel": 40859, + "chapel": 10137, + "chaplain": 38348, + "chaplin": 32545, + "chapman": 17968, + "chapp": 20634, + "chaps": 36823, + "chapter": 6014, + "chapters": 22936, + "char": 1054, + "char": 16017, + "chara": 35668, + "charac": 2792, + "character": 10997, + "character": 4009, + "characterdesign": 38149, + "characteri": 20920, + "characteristic": 44747, + "characteristics": 26037, + "characters": 6564, + "charan": 31851, + "charcoal": 19268, + "chard": 17524, + "chardon": 26599, + "chardonnay": 28161, + "charge": 25032, + "charge": 5948, + "chargeable": 35664, + "charged": 7916, + "charger": 13090, + "chargers": 17352, + "charges": 8962, + "charging": 12514, + "chariot": 38811, + "charis": 24449, + "charisma": 45041, + "charismatic": 37205, + "charitable": 23256, + "charities": 18493, + "charity": 20008, + "charity": 4607, + "charitytuesday": 42794, + "charl": 47736, + "charle": 10217, + "charles": 27983, + "charles": 5127, + "charleston": 15478, + "charley": 38027, + "charli": 21784, + "charli": 49392, + "charlie": 16764, + "charlie": 6393, + "charlotte": 18445, + "charlotte": 7871, + "charlottesville": 32027, + "charlton": 27048, + "charm": 10876, + "charmed": 39790, + "charming": 12177, + "charms": 21944, + "charred": 44085, + "chart": 42685, + "chart": 5053, + "charted": 27939, + "charter": 42345, + "charter": 13569, + "chartered": 31298, + "charters": 46626, + "charting": 39841, + "charts": 10728, + "chas": 10717, + "chas": 29838, + "chase": 21503, + "chase": 3859, + "chased": 30342, + "chaser": 29560, + "chasers": 34158, + "chases": 45011, + "chasing": 46909, + "chasing": 13376, + "chassis": 29188, + "chast": 42176, + "chasu": 41352, + "chat": 5355, + "chat": 2402, + "chatbots": 43994, + "chate": 30377, + "chateau": 44582, + "chateau": 23520, + "chath": 46849, + "chatham": 32030, + "chats": 13263, + "chatt": 21618, + "chattanoo": 28009, + "chattanooga": 29866, + "chatted": 34124, + "chatter": 33473, + "chatter": 41103, + "chatting": 12401, + "chatur": 33839, + "chau": 11263, + "chau": 37536, + "chauffe": 45440, + "chauhan": 46663, + "chav": 28997, + "chavez": 27480, + "chaw": 39639, + "chay": 45317, + "chaz": 47815, + "chc": 36233, + "chd": 41645, + "che": 983, + "che": 3842, + "chea": 39580, + "chead": 48358, + "cheap": 27036, + "cheap": 8678, + "cheape": 26164, + "cheaper": 17776, + "cheapest": 26640, + "cheat": 18180, + "cheated": 34285, + "cheating": 19722, + "chec": 1113, + "check": 7672, + "check": 1217, + "checked": 10387, + "checker": 45883, + "checkers": 48181, + "checking": 7441, + "checklist": 26989, + "checkout": 13101, + "checkpoint": 27531, + "checks": 13737, + "ched": 11341, + "ched": 2146, + "cheddar": 20551, + "chee": 5326, + "chee": 20944, + "cheek": 40000, + "cheek": 21227, + "cheeks": 23019, + "cheeky": 15068, + "cheer": 9733, + "cheer": 6918, + "cheered": 38111, + "cheerful": 28882, + "cheering": 14289, + "cheerleader": 29072, + "cheerleaders": 22343, + "cheerleading": 36366, + "cheers": 6562, + "chees": 15182, + "cheese": 10738, + "cheese": 4108, + "cheeseburger": 41200, + "cheesecake": 17803, + "cheeses": 36076, + "cheesy": 22093, + "cheetah": 27431, + "chef": 12137, + "chef": 4895, + "chefs": 14486, + "chek": 43745, + "chel": 3084, + "chel": 25970, + "chell": 46854, + "chelle": 30141, + "chelms": 34936, + "chelmsford": 39890, + "chelse": 19071, + "chelsea": 6031, + "chelseafc": 25927, + "chelten": 18889, + "cheltenham": 21589, + "chem": 5667, + "chem": 13698, + "chemi": 7179, + "chemical": 39376, + "chemical": 9208, + "chemicals": 17426, + "chemist": 23138, + "chemistry": 8841, + "chemo": 33095, + "chemo": 36348, + "chemotherapy": 41412, + "chemtrails": 46015, + "chen": 5907, + "chen": 8983, + "cheney": 43522, + "cheng": 32512, + "cheng": 30190, + "chenko": 29073, + "chennai": 28948, + "chennai": 12791, + "cheon": 11498, + "cheque": 28168, + "cher": 3597, + "cher": 3466, + "cheri": 26471, + "cherish": 20053, + "cherished": 42325, + "cherno": 35376, + "chernobyl": 40554, + "chero": 19844, + "cherokee": 22860, + "cherries": 27248, + "cherry": 21470, + "cherry": 7325, + "chers": 5789, + "chery": 38478, + "cheryl": 37784, + "cheryl": 20600, + "ches": 18346, + "ches": 1910, + "chesa": 28349, + "chesapeake": 32909, + "cheshire": 17130, + "chesney": 48747, + "chess": 27170, + "chess": 8397, + "chest": 18217, + "chest": 10563, + "chester": 10466, + "chester": 3343, + "chesterfield": 32975, + "chestnut": 21834, + "chet": 9663, + "chett": 24695, + "chev": 7152, + "chev": 41145, + "chevro": 12850, + "chevrolet": 13240, + "chevron": 33792, + "chevy": 16581, + "chew": 32645, + "chew": 22642, + "chewan": 23689, + "chewbacca": 49355, + "chewing": 31486, + "chewy": 42940, + "chey": 26968, + "chey": 31208, + "cheyenne": 34805, + "chez": 49183, + "chez": 10556, + "chf": 33021, + "chfield": 41619, + "chhat": 34127, + "chhattisgarh": 44246, + "chi": 1337, + "chi": 4039, + "chia": 19147, + "chiang": 33764, + "chibi": 22306, + "chic": 2627, + "chic": 9091, + "chica": 44190, + "chicag": 16778, + "chicago": 15038, + "chicago": 3530, + "chicagof": 40638, + "chicagofire": 46576, + "chicas": 40664, + "chichester": 43823, + "chick": 3170, + "chick": 11238, + "chicken": 26322, + "chicken": 3717, + "chickens": 21658, + "chickpea": 48109, + "chicks": 17810, + "chico": 30379, + "chie": 40046, + "chie": 12388, + "chief": 16830, + "chief": 3455, + "chiefs": 11419, + "chiev": 47761, + "chiff": 27407, + "chiffon": 31817, + "chig": 42952, + "chihu": 22857, + "chihuahu": 25437, + "chihuahua": 30181, + "chik": 45455, + "chil": 1333, + "child": 4392, + "child": 2913, + "childcare": 31133, + "childhood": 34772, + "childhood": 7551, + "childish": 31939, + "childre": 2135, + "children": 11101, + "children": 2153, + "childrens": 31551, + "childrens": 21553, + "childs": 39521, + "chile": 10022, + "chilean": 33186, + "chili": 13033, + "chill": 6498, + "chill": 6382, + "chilled": 23540, + "chillen": 45160, + "chilli": 26787, + "chilli": 17067, + "chillin": 10347, + "chilling": 10179, + "chillout": 39842, + "chills": 25460, + "chilly": 14450, + "chim": 10543, + "chimney": 26821, + "chimp": 44374, + "chin": 6555, + "chin": 8979, + "china": 38943, + "china": 2817, + "chinatown": 28582, + "chine": 4013, + "chinese": 30568, + "chinese": 4271, + "ching": 34621, + "ching": 1439, + "chino": 47181, + "chino": 27440, + "chinook": 41577, + "chinson": 33786, + "chio": 19650, + "chip": 19271, + "chip": 8730, + "chipmun": 46384, + "chipot": 17702, + "chipotle": 19284, + "chipp": 39854, + "chippe": 46541, + "chipped": 39892, + "chipping": 40323, + "chips": 8855, + "chir": 15564, + "chiro": 23413, + "chiroprac": 25987, + "chiropractic": 34437, + "chis": 19920, + "chistan": 20523, + "chiswick": 47290, + "chit": 13515, + "chit": 45626, + "chita": 49184, + "chitec": 39862, + "chive": 29222, + "chives": 34921, + "chk": 47424, + "chl": 38592, + "chley": 47748, + "chlo": 10374, + "chloe": 39966, + "chloe": 13992, + "chlor": 23135, + "chman": 35835, + "chment": 20848, + "chner": 48277, + "cho": 1327, + "cho": 5150, + "choa": 43077, + "choc": 32772, + "choc": 21983, + "choco": 46285, + "choco": 32692, + "chocol": 3443, + "chocolat": 44631, + "chocolate": 29389, + "chocolate": 3820, + "chocolates": 24120, + "choi": 23749, + "choic": 35606, + "choice": 23857, + "choice": 4051, + "choices": 11016, + "choir": 9214, + "choirs": 43277, + "choke": 30231, + "choked": 43521, + "choker": 39642, + "choking": 39993, + "chol": 19802, + "cholera": 45999, + "cholester": 26861, + "cholesterol": 27982, + "chom": 25151, + "chon": 20416, + "chon": 21601, + "chondri": 37379, + "chong": 26220, + "choo": 3869, + "choo": 24437, + "chool": 29578, + "chools": 41958, + "choose": 22756, + "choose": 5073, + "chooses": 29923, + "choosing": 13475, + "chop": 10458, + "chop": 16663, + "chopin": 42256, + "chopped": 22580, + "chopper": 24011, + "chopping": 35375, + "chopra": 24258, + "chops": 26321, + "chor": 7567, + "chor": 47795, + "choral": 26684, + "chord": 33005, + "chords": 36152, + "choreo": 17443, + "choreographer": 35952, + "choreography": 32749, + "chores": 40483, + "chori": 25718, + "chorizo": 30802, + "chorus": 20869, + "chos": 26559, + "chose": 11090, + "chosen": 10044, + "chou": 16960, + "chou": 42917, + "choudhary": 45503, + "chow": 20257, + "chow": 21657, + "chowder": 37886, + "chp": 35896, + "chr": 36918, + "chri": 1135, + "chris": 9907, + "chris": 2978, + "chrisbrown": 41035, + "chriss": 46745, + "chrissy": 44762, + "chrissy": 40485, + "christ": 1403, + "christ": 6703, + "christchurch": 27100, + "christen": 31956, + "christensen": 42226, + "christi": 3328, + "christi": 33213, + "christian": 11792, + "christian": 4729, + "christianity": 20000, + "christians": 14842, + "christie": 16084, + "christin": 30189, + "christina": 15925, + "christine": 42610, + "christine": 14712, + "christma": 12039, + "christmas": 18174, + "christmas": 1677, + "christmaseve": 44381, + "christmass": 44873, + "christop": 7917, + "christoph": 47844, + "christophe": 45486, + "christopher": 33349, + "christopher": 9630, + "christy": 28331, + "chro": 13207, + "chromatic": 44207, + "chrome": 24843, + "chrome": 9529, + "chromo": 35809, + "chron": 5577, + "chron": 39781, + "chronic": 10115, + "chronic": 13677, + "chronicle": 20034, + "chronicles": 18905, + "chrono": 29387, + "chronograph": 38397, + "chry": 13508, + "chrysler": 20078, + "chs": 40277, + "chs": 8391, + "chsnews": 44919, + "cht": 11384, + "chter": 47811, + "chu": 3799, + "chu": 13622, + "chubby": 29109, + "chuck": 13211, + "chuck": 9894, + "chuckle": 35733, + "chucky": 42026, + "chuffed": 27233, + "chuk": 25878, + "chuk": 27221, + "chul": 33001, + "chum": 46869, + "chum": 41767, + "chun": 14693, + "chun": 25391, + "chung": 28418, + "chunk": 30275, + "chunks": 45538, + "chunky": 27978, + "chups": 46331, + "chur": 2309, + "church": 14956, + "church": 2735, + "churches": 15539, + "churchill": 17527, + "chus": 36246, + "chut": 28788, + "chutney": 36261, + "chy": 15131, + "chy": 8096, + "chyna": 43398, + "châ": 48669, + "ci": 698, + "ci": 5798, + "cia": 4019, + "cial": 1143, + "cian": 32323, + "ciao": 37677, + "ciara": 31369, + "cible": 28873, + "cic": 14539, + "cic": 21517, + "cid": 27359, + "cide": 34178, + "cider": 13547, + "cides": 41326, + "cie": 19730, + "cier": 24067, + "cies": 6785, + "cif": 35698, + "cigar": 26031, + "cigar": 16525, + "cigare": 13044, + "cigarette": 18548, + "cigarettes": 22750, + "cigars": 20750, + "cii": 42408, + "cil": 9217, + "cil": 2998, + "cilan": 33998, + "cilantro": 34568, + "cili": 18977, + "ciliation": 25294, + "cim": 30021, + "cin": 2396, + "cin": 25367, + "cina": 39467, + "cincin": 13291, + "cincinnati": 14197, + "cinco": 25131, + "cincode": 40930, + "cincodemayo": 42542, + "cincy": 30015, + "cincy": 30286, + "cinde": 20660, + "cinderella": 21515, + "cindy": 34439, + "cindy": 18532, + "cine": 4015, + "cine": 27451, + "cinema": 38251, + "cinema": 6443, + "cinemas": 14845, + "cinematic": 25602, + "cinemato": 21919, + "cinematographer": 39059, + "cinematography": 33802, + "ciner": 39882, + "cing": 4014, + "cini": 25699, + "cinnam": 12768, + "cinnamon": 13460, + "cino": 18616, + "cio": 44584, + "cio": 9954, + "cion": 22024, + "ciones": 37155, + "cious": 38466, + "cip": 32884, + "cir": 2459, + "cir": 41135, + "circa": 10411, + "circle": 33574, + "circle": 7117, + "circles": 19411, + "circling": 46036, + "circu": 5143, + "circuit": 35583, + "circuit": 9801, + "circuits": 33260, + "circul": 16618, + "circular": 19733, + "circulare": 39525, + "circulareconomy": 39878, + "circulated": 46258, + "circulating": 42980, + "circulation": 27880, + "circum": 13406, + "circumstances": 18786, + "circus": 11833, + "cirque": 36049, + "cis": 9459, + "cis": 23513, + "cisco": 36689, + "cisco": 19290, + "cise": 19657, + "cisely": 33434, + "cision": 41957, + "cism": 24166, + "cist": 40906, + "cit": 4420, + "cit": 31294, + "citadel": 38036, + "citation": 33581, + "cite": 32641, + "cited": 25069, + "cites": 34490, + "citi": 4280, + "citi": 30270, + "cities": 5441, + "citing": 29088, + "citiz": 5816, + "citizen": 11720, + "citizen": 9814, + "citizens": 7949, + "citizenship": 17386, + "cito": 42636, + "citro": 27941, + "citroen": 35805, + "citrus": 17379, + "city": 5002, + "city": 1305, + "cityfc": 28751, + "cityo": 25709, + "cityof": 11595, + "cityscape": 40808, + "ciu": 39693, + "cius": 42559, + "civ": 40039, + "civic": 32240, + "civic": 11888, + "civil": 6923, + "civil": 6450, + "civilian": 21187, + "civilians": 18076, + "civilization": 22503, + "civilwar": 34524, + "ción": 44700, + "cj": 15238, + "cj": 15205, + "ck": 916, + "ck": 868, + "cke": 25224, + "cke": 40989, + "cked": 3441, + "cken": 25566, + "cker": 15509, + "cker": 4744, + "ckers": 37073, + "cket": 5525, + "ckett": 33899, + "ckey": 15029, + "ckey": 3657, + "cki": 36916, + "cki": 41055, + "cking": 4805, + "cko": 28818, + "cks": 2031, + "cky": 26229, + "cky": 3083, + "cl": 969, + "cl": 6482, + "cla": 940, + "cla": 20636, + "clad": 31606, + "cladding": 46411, + "clai": 29459, + "claim": 4290, + "claim": 6607, + "claimed": 9010, + "claiming": 15286, + "claims": 6852, + "clair": 31441, + "clair": 14039, + "claire": 20410, + "claire": 10460, + "clam": 13588, + "clam": 32598, + "clamation": 21793, + "clamp": 41501, + "clams": 38849, + "clan": 29252, + "clan": 14114, + "clancy": 37227, + "clans": 38279, + "clap": 30037, + "clap": 25546, + "clapham": 43619, + "clapton": 37683, + "clar": 3617, + "clara": 19468, + "clare": 18948, + "clare": 15927, + "claremont": 47789, + "clarence": 29320, + "clari": 15175, + "clarify": 37004, + "clarinet": 41178, + "clarity": 21323, + "clark": 13340, + "clark": 7521, + "clarke": 11548, + "clarkson": 25706, + "clas": 32003, + "clash": 38367, + "clash": 9359, + "clashes": 25193, + "clasico": 43567, + "class": 2876, + "class": 1874, + "classes": 6919, + "classi": 2507, + "classic": 9353, + "classic": 2713, + "classical": 22179, + "classical": 11355, + "classicalmusic": 27806, + "classiccar": 46906, + "classiccars": 21064, + "classics": 10634, + "classification": 26612, + "classified": 22056, + "classmate": 37090, + "classmates": 30062, + "classof": 25345, + "classroom": 9001, + "classrooms": 25768, + "classy": 11615, + "clau": 7526, + "claude": 17461, + "claudi": 39439, + "claudia": 21893, + "claudio": 31230, + "claus": 23317, + "clause": 26151, + "clave": 24111, + "claw": 49230, + "claw": 19106, + "claws": 29161, + "clay": 10402, + "clay": 8823, + "clays": 26128, + "clayton": 46445, + "clayton": 19413, + "clc": 31380, + "cle": 1321, + "cle": 2537, + "clean": 3572, + "clean": 3772, + "cleaned": 17468, + "cleanenergy": 43538, + "cleaner": 15619, + "cleaners": 33258, + "cleaning": 7210, + "cleanliness": 47886, + "cleans": 40827, + "cleanse": 28717, + "cleanser": 44170, + "cleansing": 25931, + "cleanup": 22353, + "clear": 4631, + "clear": 3143, + "clearance": 17959, + "cleared": 14880, + "clearer": 37031, + "clearing": 15481, + "clearly": 7767, + "clears": 29092, + "clearwater": 32124, + "cleary": 44342, + "cleats": 33486, + "cleavage": 44165, + "cled": 12827, + "clegg": 42915, + "clemens": 45896, + "clement": 22592, + "clement": 24714, + "clemente": 42461, + "clementine": 47112, + "clements": 49175, + "clemson": 38170, + "clemson": 19537, + "clen": 35547, + "cleo": 40344, + "cleop": 36287, + "cleopatra": 41212, + "cler": 11828, + "clergy": 42635, + "cleric": 43748, + "clerk": 22230, + "clermont": 47529, + "cles": 8077, + "cleve": 37599, + "clevel": 7701, + "cleveland": 30716, + "cleveland": 8430, + "clever": 30977, + "clever": 13385, + "clg": 47546, + "cli": 1503, + "clich": 44407, + "click": 16676, + "click": 3585, + "clicked": 29015, + "clicking": 26542, + "clicks": 31250, + "client": 48528, + "client": 7467, + "clients": 8114, + "clif": 13182, + "cliff": 23827, + "cliff": 10625, + "cliffe": 15170, + "clifford": 24226, + "cliffs": 20953, + "clifton": 23878, + "climat": 37283, + "climate": 7854, + "climate": 4589, + "climateaction": 31622, + "climatechange": 11055, + "climates": 46022, + "climax": 37033, + "climb": 7421, + "climb": 10649, + "climbed": 22528, + "climber": 36910, + "climbers": 47648, + "climbing": 9877, + "climbs": 29098, + "clin": 2879, + "clinch": 30404, + "clinched": 44064, + "cline": 37460, + "cling": 37068, + "cling": 4760, + "clinic": 7926, + "clinical": 35133, + "clinical": 9148, + "clinicians": 45866, + "clinics": 23330, + "clint": 37542, + "clint": 21160, + "clinton": 34403, + "clinton": 5820, + "clio": 46889, + "clip": 39712, + "clip": 9289, + "clipped": 45524, + "clipper": 42245, + "clippers": 23319, + "clipping": 47484, + "clips": 16594, + "clique": 34983, + "clive": 36086, + "clive": 21509, + "cll": 46091, + "cllr": 45743, + "cllr": 23034, + "clo": 1194, + "cloak": 36528, + "clock": 19878, + "clock": 6716, + "clocked": 49049, + "clocks": 25895, + "clockwise": 46150, + "clockwork": 42297, + "clon": 24477, + "clone": 22854, + "clones": 48047, + "clooney": 33161, + "clos": 48821, + "close": 10603, + "close": 2660, + "closed": 4552, + "closely": 13478, + "closer": 6377, + "closes": 11354, + "closest": 14975, + "closet": 14221, + "closeup": 35439, + "closing": 7101, + "closure": 13249, + "closures": 22923, + "cloth": 14559, + "clothes": 7080, + "clothing": 7425, + "clou": 4069, + "cloud": 12965, + "cloud": 3887, + "cloudcomputing": 41390, + "clouds": 6244, + "cloudy": 13106, + "clough": 42909, + "clover": 39574, + "clover": 22812, + "clow": 18386, + "clown": 15329, + "clowns": 30820, + "cls": 44251, + "clt": 29651, + "clt": 24236, + "clu": 996, + "club": 9642, + "club": 1736, + "clubbing": 48128, + "clubhouse": 26553, + "clubs": 9437, + "clue": 14994, + "clueless": 35350, + "clues": 23764, + "clusive": 41362, + "cluster": 15595, + "clusters": 33217, + "clut": 28507, + "clutch": 13953, + "clutter": 40804, + "cly": 12037, + "clyde": 39557, + "clyde": 18469, + "cm": 10190, + "cm": 3741, + "cma": 30554, + "cma": 31388, + "cmc": 45839, + "cmdr": 48250, + "cme": 34946, + "cmo": 24589, + "cmon": 42904, + "cmp": 46355, + "cms": 22520, + "cmt": 42727, + "cmu": 43046, + "cn": 3886, + "cn": 16200, + "cna": 48287, + "cnbc": 41242, + "cnbc": 24371, + "cnblue": 36018, + "cnc": 20571, + "cnet": 47487, + "cnews": 24319, + "cng": 41496, + "cnn": 22405, + "cnn": 8259, + "cns": 46095, + "cny": 31614, + "co": 622, + "co": 1320, + "coa": 29167, + "coach": 3275, + "coach": 2312, + "coached": 30228, + "coachella": 20222, + "coaches": 6924, + "coaching": 7766, + "coal": 10227, + "coal": 7919, + "coalition": 12920, + "coast": 6398, + "coast": 3720, + "coastal": 38246, + "coastal": 10852, + "coaster": 15944, + "coasters": 31548, + "coastguard": 40601, + "coastline": 27959, + "coasts": 42225, + "coat": 28869, + "coat": 7356, + "coated": 23401, + "coates": 36899, + "coating": 25369, + "coatings": 48706, + "coats": 18075, + "cob": 20140, + "cob": 32863, + "cobain": 36866, + "cobalt": 30896, + "cobb": 22719, + "cobble": 47894, + "cobra": 21574, + "coc": 23036, + "coc": 39498, + "coca": 21197, + "cocac": 26393, + "cocacola": 31248, + "cocaine": 20534, + "coch": 18599, + "cochran": 48798, + "cochrane": 41752, + "coco": 11850, + "coco": 13316, + "cocoa": 18074, + "cocon": 8597, + "coconut": 9581, + "cod": 16132, + "cod": 11915, + "code": 11582, + "code": 3217, + "coded": 33703, + "coden": 43914, + "coder": 41561, + "codes": 14566, + "codi": 39711, + "coding": 12647, + "cody": 23222, + "cody": 12666, + "coe": 15386, + "coed": 41028, + "coel": 45633, + "coer": 41198, + "coeur": 44986, + "coffe": 2255, + "coffee": 12898, + "coffee": 2453, + "coffees": 41184, + "coffey": 48066, + "cofficial": 18757, + "coffin": 29907, + "cog": 26362, + "cog": 35960, + "cogn": 12210, + "cognac": 44361, + "cognition": 46825, + "cognitive": 16584, + "cohe": 20669, + "cohen": 13381, + "coherent": 48450, + "cohort": 22782, + "coil": 25307, + "coim": 41528, + "coin": 14651, + "coin": 4170, + "coinci": 14015, + "coincidence": 19807, + "coins": 10530, + "coke": 39602, + "coke": 14035, + "col": 754, + "col": 9371, + "cola": 15444, + "colbert": 31647, + "colby": 32068, + "colchester": 31715, + "cold": 11146, + "cold": 3153, + "colder": 23859, + "coldest": 31438, + "coldplay": 27770, + "cole": 9305, + "cole": 8166, + "coleman": 15774, + "coles": 40265, + "coles": 30398, + "coli": 18877, + "coli": 15910, + "colin": 20989, + "colin": 10238, + "coliseum": 21836, + "coll": 25982, + "coll": 23898, + "colla": 2929, + "collab": 14013, + "collabor": 4437, + "collaborate": 21271, + "collaborated": 42265, + "collaborating": 25545, + "collaboration": 6642, + "collaborations": 36520, + "collaborative": 15841, + "collaborator": 48186, + "collaborators": 45901, + "collage": 11258, + "collagen": 36120, + "collap": 16881, + "collapse": 16520, + "collapsed": 25037, + "collapses": 43601, + "collar": 39662, + "collar": 13497, + "collateral": 44512, + "colle": 1801, + "colleague": 13067, + "colleagues": 8203, + "collec": 1733, + "collect": 10186, + "collected": 11980, + "collecti": 18530, + "collectible": 25680, + "collectibles": 21519, + "collecting": 10325, + "collection": 2548, + "collections": 12760, + "collective": 10162, + "collectively": 40687, + "collector": 13522, + "collectors": 20540, + "collects": 31576, + "colleen": 31020, + "college": 13512, + "college": 2229, + "colleges": 17357, + "collegi": 16311, + "collegiate": 18068, + "colli": 8262, + "collide": 27214, + "collie": 30611, + "collier": 35748, + "collin": 24056, + "collin": 32116, + "colling": 32319, + "collingwood": 45873, + "collins": 8684, + "collision": 15407, + "collo": 25115, + "colloqui": 37243, + "colloquium": 46514, + "collu": 25658, + "collusion": 33864, + "colo": 7300, + "colo": 27288, + "cologne": 22216, + "cology": 19187, + "colom": 8987, + "colombia": 12901, + "colombian": 28701, + "colombo": 33207, + "colon": 8280, + "colon": 29050, + "colonel": 22674, + "coloni": 22667, + "colonial": 16530, + "colonialism": 43385, + "colonies": 38738, + "colony": 18767, + "color": 4036, + "color": 3140, + "colorado": 34580, + "colorado": 6742, + "colorec": 41171, + "colored": 11775, + "colorful": 11444, + "colori": 28764, + "coloring": 17696, + "colorized": 46730, + "colors": 5389, + "colorstv": 28195, + "colorway": 44576, + "colossal": 40258, + "colosse": 48142, + "colossus": 34022, + "colour": 10240, + "colour": 4769, + "coloured": 17111, + "colourful": 15562, + "colouring": 31803, + "colours": 7626, + "cols": 35726, + "colt": 19726, + "colton": 32249, + "coltrane": 42333, + "colts": 16135, + "colum": 4164, + "columb": 31043, + "columbi": 25947, + "columbia": 9410, + "columbus": 11273, + "column": 10593, + "columnist": 28958, + "columns": 29056, + "com": 610, + "com": 2464, + "coma": 19620, + "comb": 3587, + "comb": 16380, + "combat": 35083, + "combat": 9275, + "combating": 46121, + "combe": 14363, + "combin": 25112, + "combination": 11312, + "combinations": 34950, + "combine": 12919, + "combined": 10427, + "combines": 22991, + "combining": 23561, + "combo": 10155, + "combos": 48117, + "combs": 30694, + "combu": 35629, + "combustion": 44654, + "comcast": 30043, + "come": 4225, + "come": 891, + "comeback": 8234, + "comedian": 13848, + "comedians": 33758, + "comedic": 43360, + "comedy": 19346, + "comedy": 4749, + "comer": 42997, + "comer": 20916, + "comers": 34436, + "comes": 2091, + "comet": 21405, + "comets": 40636, + "comey": 22957, + "comfor": 6563, + "comfort": 44000, + "comfort": 7808, + "comfortable": 8652, + "comfortably": 30392, + "comforting": 33835, + "comforts": 42243, + "comfy": 15736, + "comi": 40781, + "comic": 7729, + "comic": 4962, + "comicart": 46018, + "comicbook": 46564, + "comicbooks": 22018, + "comiccon": 18379, + "comicon": 43820, + "comics": 4256, + "comin": 18164, + "coming": 14916, + "coming": 1171, + "comingsoon": 19894, + "comm": 965, + "comm": 11413, + "comman": 39780, + "command": 18391, + "command": 11350, + "commander": 11265, + "commanders": 41667, + "commanding": 36933, + "commandments": 43409, + "commando": 31361, + "commands": 38163, + "comme": 29692, + "commemor": 9495, + "commemorate": 21242, + "commemorates": 45149, + "commemorating": 28734, + "commemoration": 29288, + "commemorative": 24623, + "commen": 15795, + "commence": 25059, + "commenced": 43908, + "commencement": 21666, + "commences": 48551, + "commend": 37555, + "commended": 40702, + "comment": 20035, + "comment": 5761, + "commentary": 14146, + "commentator": 32016, + "commented": 28328, + "commenting": 37292, + "comments": 6606, + "commer": 4028, + "commerce": 8333, + "commerci": 15601, + "commercial": 31802, + "commercial": 6287, + "commercials": 30724, + "commish": 45399, + "commissi": 6000, + "commission": 5292, + "commissioned": 16565, + "commissioner": 10221, + "commissioners": 30702, + "commissioning": 29585, + "commissions": 20668, + "commit": 3041, + "commit": 11797, + "commitment": 7770, + "commitments": 32136, + "commits": 20241, + "committed": 7907, + "committee": 5636, + "committees": 40504, + "committing": 21937, + "commod": 9496, + "commodities": 30350, + "commodity": 29041, + "commodore": 31129, + "common": 8414, + "common": 4176, + "commonly": 20344, + "commons": 16653, + "commonwealth": 16569, + "comms": 18832, + "commu": 9561, + "commun": 1515, + "communal": 32809, + "communi": 16164, + "communic": 4784, + "communicate": 19809, + "communication": 7999, + "communications": 10052, + "communion": 28579, + "communism": 35387, + "communist": 18602, + "communities": 6361, + "community": 14784, + "community": 1927, + "commute": 15898, + "commuter": 27782, + "commuters": 30823, + "commuting": 43503, + "como": 16236, + "comp": 2561, + "comp": 11679, + "compac": 40014, + "compact": 13690, + "compan": 1995, + "companies": 5361, + "companion": 14963, + "companions": 37124, + "company": 2634, + "compar": 7580, + "comparable": 27092, + "comparative": 33388, + "compare": 13771, + "compared": 10544, + "compares": 25104, + "comparing": 20564, + "comparison": 14186, + "comparisons": 40870, + "compart": 30072, + "compartment": 40383, + "compass": 19438, + "compassion": 14463, + "compassionate": 30193, + "compati": 17295, + "compatibility": 41614, + "compatible": 21286, + "compe": 5254, + "compelled": 49375, + "compelling": 21766, + "compen": 42079, + "compens": 15172, + "compensation": 18663, + "compet": 2932, + "compete": 10038, + "competed": 27767, + "competen": 31853, + "competence": 31165, + "competency": 49293, + "competent": 28113, + "competes": 39826, + "competing": 13068, + "competit": 15892, + "competiti": 32581, + "competition": 3742, + "competitions": 23259, + "competitive": 10687, + "competitiveness": 43209, + "competitor": 26633, + "competitors": 23638, + "compilation": 20446, + "compiled": 34579, + "compla": 7428, + "complain": 19292, + "complained": 42029, + "complaining": 20812, + "complains": 46363, + "complaint": 20391, + "complaints": 20020, + "comple": 1730, + "complement": 36624, + "complementary": 48953, + "complete": 3263, + "completed": 5976, + "completely": 5989, + "completes": 19321, + "completing": 14949, + "completion": 15915, + "complex": 16099, + "complex": 6324, + "complexes": 47870, + "complexion": 47732, + "complexity": 24815, + "compli": 5270, + "compliance": 14658, + "compliant": 29893, + "complic": 11460, + "complicated": 16621, + "complications": 29936, + "compliment": 25116, + "complimentary": 20948, + "compliments": 25477, + "comply": 36281, + "component": 21284, + "components": 16816, + "compos": 7783, + "compose": 43659, + "composed": 19916, + "composer": 12104, + "composers": 33314, + "composing": 40412, + "composite": 21606, + "composites": 45395, + "composition": 17510, + "compositions": 44652, + "compost": 46002, + "compost": 33307, + "compound": 19980, + "compounds": 33991, + "compre": 8483, + "compreh": 42976, + "comprehen": 12050, + "comprehend": 48230, + "comprehensive": 13854, + "compress": 33353, + "compressed": 42359, + "compression": 25638, + "compressor": 39607, + "compri": 29445, + "compromise": 26611, + "compromised": 38576, + "compromising": 45436, + "comps": 48665, + "compton": 28364, + "compu": 11639, + "compul": 25869, + "compulsory": 39345, + "computing": 12732, + "comra": 25553, + "comrade": 30844, + "comrades": 29282, + "coms": 30493, + "con": 616, + "con": 2457, + "cona": 30605, + "conan": 24750, + "conce": 9145, + "concealed": 35419, + "conceded": 37895, + "conceived": 39725, + "concentr": 11085, + "concentrate": 30846, + "concentrated": 36776, + "concentration": 18565, + "concep": 8389, + "concepcion": 47035, + "concept": 6353, + "conceptart": 31162, + "conception": 30510, + "conceptions": 40307, + "concepts": 16763, + "conceptu": 42745, + "conceptual": 34070, + "concer": 2228, + "concern": 12928, + "concerned": 12020, + "concerning": 21772, + "concerns": 11134, + "concert": 32180, + "concert": 3066, + "concerto": 24710, + "concerts": 14418, + "concession": 38117, + "concessions": 43981, + "concier": 28859, + "concierge": 39850, + "conclave": 38098, + "conclu": 9627, + "conclude": 37525, + "concluded": 27825, + "concludes": 30634, + "conclusion": 20932, + "conclusions": 39507, + "conco": 43034, + "concor": 19913, + "concord": 26448, + "concordia": 35492, + "concours": 36282, + "concourse": 37793, + "concre": 43658, + "concrete": 9637, + "concussion": 28321, + "condem": 13287, + "condemn": 27212, + "condemned": 35145, + "condemns": 32092, + "conden": 24816, + "conditi": 11170, + "condition": 36978, + "condition": 7336, + "conditional": 24671, + "conditioned": 37014, + "conditioner": 31239, + "conditioning": 18181, + "conditions": 5892, + "condo": 19952, + "condol": 18661, + "condolences": 20836, + "condom": 39021, + "condomin": 42589, + "condoms": 37878, + "condor": 47643, + "condos": 42342, + "condu": 40772, + "conduc": 5379, + "conduct": 11647, + "conducted": 13080, + "conducting": 16787, + "conductor": 22317, + "conducts": 32084, + "cone": 39279, + "cone": 10266, + "cones": 26718, + "coney": 41837, + "conf": 6477, + "confe": 1968, + "confeder": 17104, + "confederate": 24864, + "confederation": 43484, + "conferen": 37961, + "conference": 2230, + "conferences": 22811, + "conferencing": 47320, + "confess": 38860, + "confession": 22572, + "confessions": 29404, + "confetti": 37923, + "confi": 5005, + "confidence": 8510, + "confident": 12365, + "confidential": 28712, + "configu": 46746, + "configur": 26950, + "configuration": 33378, + "confin": 45316, + "confined": 40973, + "confir": 3930, + "confirm": 12130, + "confirmation": 19645, + "confirmed": 6346, + "confirming": 38433, + "confirms": 11803, + "confis": 36285, + "confit": 42241, + "confl": 8173, + "conflic": 19029, + "conflict": 10397, + "conflicting": 43894, + "conflicts": 28713, + "confor": 40933, + "confron": 20033, + "confront": 38382, + "confrontation": 41478, + "confu": 6890, + "confuse": 37503, + "confused": 10946, + "confusing": 24683, + "confusion": 20493, + "cong": 24407, + "conge": 20013, + "congestion": 24432, + "congo": 20334, + "congr": 1227, + "congrats": 1887, + "congratul": 1750, + "congratulate": 16633, + "congratulated": 42004, + "congratulates": 24580, + "congratulating": 30967, + "congratulation": 24751, + "congratulations": 1864, + "congre": 7947, + "congreg": 40727, + "congregation": 32618, + "congress": 12452, + "congress": 4599, + "congressional": 15239, + "congressman": 17145, + "congresswoman": 37317, + "coni": 39031, + "coni": 36651, + "conj": 41543, + "conju": 33821, + "conjunction": 34226, + "conley": 44536, + "conline": 37593, + "conn": 41836, + "conn": 20329, + "conne": 8437, + "connec": 29933, + "connect": 19969, + "connected": 27506, + "connecting": 41429, + "connection": 26840, + "connections": 37161, + "connie": 25739, + "connoisse": 46012, + "connol": 27739, + "connolly": 29537, + "connor": 21984, + "connor": 10218, + "conom": 2664, + "conomy": 22529, + "conor": 29955, + "conor": 19478, + "conqu": 13382, + "conquer": 38585, + "conquer": 19821, + "conquered": 27099, + "conquering": 43778, + "conquest": 35367, + "conrad": 22073, + "cons": 10311, + "consci": 9427, + "conscience": 27310, + "conscious": 14914, + "consciously": 46755, + "consciousness": 17894, + "conse": 34887, + "consecu": 12084, + "consecutive": 12413, + "consen": 23110, + "consensus": 25071, + "consent": 21922, + "consequ": 13003, + "consequence": 42262, + "consequences": 15682, + "conserv": 4649, + "conservancy": 46729, + "conservation": 37616, + "conservation": 8322, + "conservative": 11421, + "conservatives": 17631, + "conservatory": 32140, + "conserve": 34231, + "consi": 2899, + "consider": 12471, + "consider": 6734, + "considerable": 38256, + "considerably": 38510, + "consideration": 24310, + "considerations": 33700, + "considered": 9487, + "considering": 10761, + "considers": 24691, + "consist": 10410, + "consist": 33735, + "consisted": 49354, + "consistency": 25683, + "consistent": 16439, + "consistently": 23799, + "consisting": 39241, + "consists": 23458, + "consol": 27869, + "consolation": 38888, + "console": 13403, + "consoles": 33136, + "consoli": 21586, + "consolidation": 41111, + "consor": 27108, + "consortium": 29988, + "conspir": 12680, + "conspiracy": 15236, + "const": 3826, + "constable": 29179, + "constan": 38718, + "constance": 40682, + "constant": 32000, + "constant": 13111, + "constantine": 30640, + "constantly": 14336, + "constell": 21913, + "constellation": 25991, + "constitu": 6299, + "constituency": 22464, + "constituents": 32075, + "constitution": 12157, + "constitutional": 16091, + "constra": 28973, + "constraints": 41910, + "constru": 3983, + "construc": 13321, + "construct": 24467, + "constructed": 16876, + "constructing": 33653, + "construction": 48873, + "construction": 4585, + "constructive": 31810, + "consu": 4689, + "consul": 5295, + "consul": 33630, + "consulate": 34341, + "consult": 9438, + "consult": 26727, + "consultancy": 31735, + "consultant": 14196, + "consultants": 27203, + "consultation": 15777, + "consultations": 43424, + "consulting": 15883, + "consume": 28919, + "consumed": 29653, + "consumer": 34408, + "consumer": 10422, + "consumers": 14014, + "consuming": 30607, + "consumption": 14904, + "cont": 2036, + "cont": 21425, + "contact": 39367, + "contact": 3523, + "contacted": 37331, + "contacts": 22789, + "contag": 29259, + "contagious": 33984, + "contain": 9948, + "contain": 15187, + "contained": 23836, + "container": 14913, + "containers": 20448, + "containing": 20281, + "contains": 12844, + "contamin": 24662, + "contaminated": 35773, + "contamination": 31770, + "conte": 15402, + "conte": 26882, + "contempl": 21924, + "contemplating": 33854, + "contempor": 14538, + "contemporary": 16607, + "contemporary": 8859, + "contemporaryart": 20212, + "contempt": 39293, + "conten": 42201, + "contender": 23573, + "contenders": 29711, + "content": 15526, + "content": 4750, + "contentmarketing": 20429, + "contents": 14850, + "contest": 23103, + "contest": 4576, + "contestalert": 27313, + "contestant": 25682, + "contestants": 28062, + "contested": 37845, + "contests": 32210, + "contex": 42015, + "context": 13089, + "conti": 46431, + "conti": 40842, + "contin": 1918, + "continent": 19623, + "continental": 14089, + "continents": 38642, + "conting": 27104, + "contingent": 36467, + "continu": 4688, + "continually": 34086, + "continuation": 38964, + "continue": 3942, + "continued": 10150, + "continues": 4305, + "continuing": 11009, + "continuity": 34035, + "continuous": 17033, + "continuously": 29634, + "continuum": 44978, + "contour": 34733, + "contr": 22871, + "contra": 9880, + "contra": 38620, + "contrac": 7581, + "contracep": 35109, + "contract": 6120, + "contracting": 39091, + "contractor": 21429, + "contractors": 22427, + "contracts": 16563, + "contradic": 27957, + "contrary": 32805, + "contrast": 18501, + "contrasting": 40758, + "contribu": 4753, + "contribute": 14112, + "contributed": 19397, + "contributes": 34203, + "contributing": 21762, + "contribution": 11116, + "contributions": 14465, + "contributor": 24553, + "contributors": 32908, + "contro": 2372, + "control": 9963, + "control": 3366, + "controlled": 14140, + "controller": 12929, + "controllers": 30374, + "controlling": 26427, + "controls": 15746, + "controversi": 13674, + "controversial": 14617, + "controversy": 18659, + "conv": 48382, + "conve": 18421, + "conven": 7283, + "conveni": 33278, + "convenience": 17859, + "convenient": 18978, + "conveniently": 40844, + "convention": 6752, + "conventional": 20835, + "conventions": 41404, + "conver": 6336, + "convergence": 35381, + "convers": 4577, + "conversation": 5690, + "conversations": 12326, + "converse": 24149, + "conversion": 15111, + "conversions": 44137, + "convert": 20074, + "converted": 20808, + "converter": 34611, + "convertible": 19608, + "converting": 34674, + "converts": 42470, + "convey": 38342, + "convic": 11150, + "convicted": 18668, + "conviction": 24967, + "convictions": 44366, + "convin": 12889, + "convince": 20351, + "convinced": 17388, + "convincing": 27742, + "convo": 19372, + "convocation": 30674, + "convos": 44842, + "convoy": 30292, + "conway": 21410, + "conwy": 48971, + "cony": 14501, + "coo": 1664, + "coo": 21691, + "coogs": 47624, + "cook": 9726, + "cook": 5977, + "cookbook": 21086, + "cooke": 29979, + "cooked": 11452, + "cooker": 23806, + "cookery": 38779, + "cookie": 9367, + "cookies": 8320, + "cookin": 46610, + "cooking": 39248, + "cooking": 6283, + "cookout": 39743, + "cooks": 24256, + "cool": 5594, + "cool": 2077, + "cooled": 37170, + "cooler": 11078, + "coolest": 10566, + "cooling": 15291, + "coom": 41726, + "coon": 34260, + "coon": 16958, + "coop": 39917, + "coop": 18910, + "cooper": 7264, + "cooper": 8133, + "cooperate": 42936, + "cooperation": 11785, + "cooperative": 24517, + "coops": 48531, + "coordin": 8187, + "coordinate": 38250, + "coordinated": 32540, + "coordinating": 40075, + "coordination": 25611, + "coordinator": 13967, + "coors": 36025, + "cop": 3196, + "cop": 7070, + "copa": 22749, + "copd": 45876, + "cope": 47635, + "cope": 12564, + "copeland": 37604, + "copen": 15637, + "copenhagen": 17390, + "coper": 41891, + "copernic": 45519, + "copied": 36770, + "copies": 9851, + "coping": 30545, + "copolitics": 45846, + "copp": 20937, + "copped": 42229, + "copper": 24741, + "copper": 10333, + "coppola": 47427, + "cops": 10719, + "copter": 28049, + "copy": 11376, + "copy": 4509, + "copying": 38925, + "copyright": 15778, + "cor": 851, + "cor": 18559, + "cora": 34953, + "coral": 31220, + "coral": 12054, + "corbett": 35699, + "corbin": 35578, + "corbyn": 14026, + "cord": 40893, + "cord": 11181, + "corden": 41999, + "cordi": 41681, + "cordless": 44412, + "cords": 22164, + "core": 19622, + "core": 5000, + "cores": 37874, + "corey": 31279, + "corey": 15288, + "corgi": 31320, + "cori": 26508, + "coriander": 37491, + "corin": 17716, + "corinthians": 34471, + "cork": 18148, + "cork": 10376, + "corn": 5202, + "corn": 5894, + "cornelius": 45865, + "cornell": 38689, + "cornell": 20859, + "corner": 18509, + "corner": 5253, + "corners": 19584, + "cornerstone": 36280, + "cornish": 23774, + "cornwall": 37903, + "cornwall": 10777, + "coron": 13210, + "corona": 25564, + "coronado": 43946, + "coronary": 45955, + "coronation": 25014, + "coroner": 47241, + "corp": 29203, + "corp": 10918, + "corpor": 4258, + "corporal": 42445, + "corporate": 33877, + "corporate": 6838, + "corporation": 11282, + "corporations": 25482, + "corps": 11330, + "corpse": 29408, + "corpus": 31672, + "correc": 5011, + "correct": 8340, + "corrected": 35628, + "correction": 20843, + "correctional": 38030, + "corrections": 37507, + "correctly": 15359, + "correlation": 29218, + "correspon": 20203, + "correspondent": 29996, + "corri": 12974, + "corridor": 20592, + "corrie": 23961, + "corro": 24936, + "corro": 42033, + "corrosion": 39191, + "corru": 6501, + "corrup": 30429, + "corrupt": 15194, + "corruption": 9141, + "corsa": 47670, + "corsair": 42367, + "corset": 40408, + "cortex": 40109, + "cortez": 30461, + "corvette": 24367, + "cory": 23221, + "cory": 18329, + "cos": 5865, + "cos": 5700, + "cosby": 30324, + "cosc": 45944, + "coscino": 47909, + "cose": 26495, + "cosm": 37486, + "cosme": 9628, + "cosmetic": 23918, + "cosmetics": 12896, + "cosmic": 47398, + "cosmic": 18304, + "cosmo": 12829, + "cosmo": 32072, + "cosmopolitan": 35518, + "cosmos": 22151, + "cospla": 15149, + "cosplay": 42401, + "cosplay": 6435, + "cosplayer": 30215, + "cosplaying": 46701, + "cost": 11360, + "cost": 4713, + "costa": 10480, + "costar": 28659, + "costarica": 31272, + "costco": 31045, + "costello": 30667, + "costing": 39193, + "costly": 30170, + "costs": 7628, + "costu": 5786, + "costume": 7235, + "costumes": 15150, + "cosy": 22848, + "cot": 4718, + "cot": 5871, + "cote": 44234, + "cote": 20751, + "cotland": 32576, + "cotsw": 23303, + "cotswolds": 35546, + "cott": 8211, + "cott": 11349, + "cottage": 12155, + "cottages": 34405, + "cotton": 22218, + "cotton": 7050, + "cou": 1368, + "couch": 12724, + "cougar": 35028, + "cougar": 27042, + "cougars": 20425, + "cough": 35631, + "cough": 18498, + "cougs": 28482, + "coul": 22483, + "could": 44812, + "could": 1510, + "couldn": 4072, + "couldnt": 29042, + "coulter": 42291, + "coun": 939, + "counc": 12927, + "council": 18187, + "council": 3620, + "councill": 15732, + "councillor": 21179, + "councillors": 29695, + "councilman": 40833, + "councils": 29938, + "counsel": 13780, + "counsel": 19814, + "counseling": 25000, + "counsell": 47510, + "counselling": 40581, + "counselor": 26148, + "counselors": 38688, + "count": 6073, + "count": 5887, + "countdown": 39559, + "countdown": 7500, + "counted": 23149, + "counter": 10134, + "counter": 7352, + "counterfe": 33067, + "counterfeit": 44242, + "counterpart": 39216, + "counterparts": 42106, + "counters": 46170, + "countess": 46276, + "counties": 12338, + "counting": 9723, + "countless": 21819, + "countries": 5489, + "country": 7896, + "country": 2157, + "countryfile": 47023, + "countrymusic": 30372, + "countryside": 16303, + "counts": 12264, + "county": 18734, + "county": 2116, + "coup": 9871, + "coup": 16479, + "coupe": 16773, + "couple": 40136, + "couple": 3377, + "coupled": 37153, + "couples": 14752, + "coupling": 45595, + "coupon": 14019, + "coupons": 23945, + "cour": 1391, + "coura": 4436, + "courage": 9828, + "courageous": 25005, + "courier": 27217, + "cours": 21493, + "course": 43225, + "course": 2613, + "courses": 9464, + "court": 16837, + "court": 2908, + "courte": 5088, + "courtesy": 5228, + "courthouse": 22205, + "courtney": 33601, + "courtney": 15990, + "courtroom": 41071, + "courts": 13514, + "courty": 20121, + "courtyard": 21900, + "cous": 48397, + "cousin": 7780, + "cousins": 14073, + "cout": 29118, + "coutinho": 35530, + "couture": 14808, + "cov": 19384, + "cov": 48385, + "cove": 21700, + "cove": 14708, + "coven": 12483, + "covenant": 29647, + "coventry": 18007, + "cover": 13534, + "cover": 2202, + "coverage": 6810, + "covered": 5603, + "covering": 9462, + "covers": 7745, + "covert": 40134, + "coveted": 36119, + "covington": 43196, + "cow": 5076, + "cow": 9706, + "cowan": 42699, + "coward": 33729, + "cowards": 48972, + "cowboy": 25833, + "cowboy": 13657, + "cowboys": 11864, + "cowboysnation": 43082, + "cowell": 39015, + "cowgirl": 47090, + "coworker": 30727, + "coworkers": 30821, + "coworking": 36034, + "cows": 15204, + "cowx": 23831, + "cox": 25784, + "cox": 11597, + "coy": 12765, + "coy": 15742, + "coyi": 48407, + "coyle": 45348, + "coyne": 44729, + "coyo": 16614, + "coyote": 26586, + "coyotes": 30423, + "coys": 19736, + "coz": 39922, + "coz": 14282, + "cozy": 14873, + "cp": 7905, + "cp": 9130, + "cpa": 30095, + "cpac": 45731, + "cpc": 26125, + "cpd": 23402, + "cpec": 48007, + "cpfc": 27553, + "cpi": 41795, + "cpl": 26852, + "cpr": 25134, + "cps": 27078, + "cpt": 32892, + "cpu": 27700, + "cq": 48910, + "cq": 48417, + "cr": 1075, + "cr": 3483, + "cra": 1184, + "cra": 18362, + "crab": 27382, + "crab": 11574, + "crabs": 30908, + "crack": 11222, + "crack": 10334, + "crackdown": 29527, + "cracked": 19826, + "cracker": 16298, + "crackers": 26200, + "cracking": 13008, + "cracks": 21426, + "cracy": 24749, + "cradle": 29384, + "crae": 40438, + "craf": 10873, + "craft": 7717, + "craft": 3588, + "craftbeer": 12371, + "crafted": 12424, + "crafthour": 42324, + "crafting": 26886, + "crafts": 33276, + "crafts": 13383, + "craftsman": 39528, + "craftsmanship": 36682, + "crafty": 32317, + "craic": 46962, + "craig": 14042, + "craig": 8061, + "craigslist": 43865, + "cram": 29809, + "cramer": 44592, + "cramps": 46106, + "cran": 7761, + "cranberries": 49361, + "cranberry": 23824, + "crane": 14626, + "cranes": 26979, + "crani": 45674, + "crank": 46246, + "crank": 32283, + "cranston": 44340, + "crap": 11899, + "crappy": 30475, + "crash": 37150, + "crash": 5033, + "crashed": 16638, + "crashes": 17013, + "crashing": 24991, + "crat": 46696, + "crate": 24756, + "crater": 22663, + "crates": 30172, + "cratic": 32175, + "crative": 39999, + "crats": 43056, + "crave": 33397, + "craven": 33625, + "craving": 18344, + "cravings": 34476, + "craw": 7400, + "crawfish": 42772, + "crawford": 15918, + "crawl": 20106, + "crawler": 41012, + "crawley": 42316, + "crawling": 37066, + "cray": 24184, + "cray": 27032, + "crayon": 41801, + "crayons": 43508, + "craz": 25776, + "craze": 30637, + "craziest": 32690, + "craziness": 46436, + "crazy": 17540, + "crazy": 3578, + "crc": 25618, + "cre": 798, + "cre": 17762, + "cream": 23184, + "cream": 3867, + "creams": 41447, + "creamy": 17206, + "crease": 48441, + "create": 30949, + "create": 3380, + "created": 4080, + "creates": 10361, + "creati": 6714, + "creating": 5524, + "creation": 38293, + "creation": 6900, + "creations": 17411, + "creative": 15237, + "creative": 4450, + "creatives": 29352, + "creativity": 9636, + "creator": 10173, + "creators": 17981, + "creature": 14317, + "creatures": 13938, + "cred": 7314, + "cred": 22377, + "credenti": 29487, + "credentials": 33422, + "credi": 21097, + "credibility": 34984, + "credible": 32983, + "credit": 21467, + "credit": 3900, + "credited": 32480, + "credits": 10654, + "creds": 43462, + "cree": 33961, + "cree": 36014, + "creed": 18845, + "creek": 26120, + "creek": 5526, + "creep": 8153, + "creep": 26084, + "creeper": 38662, + "creeping": 29697, + "creeps": 45135, + "creepy": 11943, + "creighton": 42823, + "creme": 22681, + "creole": 45632, + "crepe": 38611, + "crescent": 18211, + "cress": 39124, + "crest": 35985, + "crest": 15760, + "crested": 36656, + "crete": 8584, + "crew": 21560, + "crew": 3462, + "crewe": 43284, + "crews": 10463, + "cri": 1621, + "cri": 38962, + "crib": 23271, + "cric": 4328, + "cricke": 19098, + "cricket": 21859, + "cricket": 5373, + "cricketer": 28439, + "cricketers": 43986, + "cried": 15290, + "cries": 19769, + "crime": 13872, + "crime": 4896, + "crimea": 28614, + "crimes": 11827, + "crimin": 5874, + "criminal": 30197, + "criminal": 8255, + "criminals": 18783, + "crimson": 19437, + "cringe": 42588, + "cripp": 33588, + "cris": 37818, + "crises": 36403, + "crisis": 5712, + "crisp": 15145, + "crispr": 39784, + "crisps": 35744, + "crispy": 16458, + "criss": 29708, + "cristi": 12699, + "cristian": 48808, + "cristiano": 14807, + "cristina": 33395, + "cristo": 38315, + "crit": 3613, + "crit": 48130, + "criteri": 33627, + "criteria": 24849, + "criterion": 43841, + "criti": 25333, + "critic": 12417, + "critic": 19361, + "critical": 15314, + "critical": 6808, + "critically": 21570, + "criticalrole": 33606, + "criticalrole": 22742, + "criticalrolefanart": 43663, + "critici": 20333, + "criticism": 17405, + "criticize": 46081, + "criticized": 41557, + "critics": 16946, + "critique": 32982, + "critters": 35423, + "crm": 22610, + "cro": 1192, + "cro": 22522, + "croati": 28072, + "croatia": 13323, + "croatian": 34795, + "croc": 43350, + "croche": 35352, + "crochet": 17554, + "crock": 41685, + "crocker": 47843, + "crockett": 48313, + "crocod": 24519, + "crocodile": 24757, + "crocs": 38988, + "croft": 16657, + "croissant": 46011, + "croix": 44735, + "crom": 25082, + "crombie": 46162, + "cromwell": 45345, + "cron": 17361, + "croo": 16443, + "crook": 43744, + "crooked": 48473, + "crooked": 25644, + "crooks": 44226, + "crop": 40751, + "crop": 9955, + "cropped": 31139, + "crops": 16290, + "crore": 18274, + "crores": 37281, + "cros": 16670, + "crosby": 21095, + "cross": 5266, + "cross": 3417, + "crossed": 11731, + "crosses": 20473, + "crossfit": 47214, + "crossfit": 20395, + "crossing": 8673, + "crossings": 43517, + "crossover": 17194, + "crossroads": 27427, + "crossword": 32945, + "crou": 31206, + "crouch": 36506, + "crow": 3138, + "crow": 16019, + "crowd": 12036, + "crowd": 4570, + "crowded": 20182, + "crowdfunding": 17971, + "crowds": 16092, + "crowe": 33560, + "crowley": 32287, + "crown": 22190, + "crown": 6902, + "crowned": 16109, + "crowns": 33229, + "crows": 27134, + "croy": 21676, + "croydon": 27116, + "crs": 28449, + "crt": 43877, + "cru": 1815, + "cru": 29788, + "cruci": 18499, + "crucial": 12396, + "crude": 20677, + "cruel": 16073, + "cruel": 17573, + "cruelty": 20675, + "cruis": 27721, + "cruise": 36425, + "cruise": 6764, + "cruiser": 21394, + "cruises": 19214, + "cruising": 19743, + "crum": 43268, + "crumb": 48327, + "crumb": 39909, + "crumble": 36595, + "crumbs": 35893, + "crun": 17407, + "crunch": 16620, + "crunchy": 31366, + "crusad": 19133, + "crusade": 36846, + "crusader": 40171, + "crusaders": 31319, + "crush": 22296, + "crush": 7610, + "crushed": 18270, + "crusher": 44923, + "crushes": 35844, + "crushing": 20790, + "crust": 23136, + "crusted": 37314, + "cruz": 33689, + "cruz": 8403, + "cry": 2837, + "cry": 6290, + "crying": 6828, + "cryo": 32215, + "cryp": 4865, + "crypt": 37814, + "cryptic": 46925, + "crypto": 8080, + "crypto": 9608, + "cryptocurrencies": 33329, + "cryptocurrency": 12070, + "cryst": 15891, + "crystal": 17387, + "crystal": 6517, + "crystalli": 47551, + "crystals": 18350, + "cs": 11978, + "cs": 2804, + "csa": 26355, + "csc": 41727, + "csc": 37266, + "csd": 36913, + "cse": 41659, + "csg": 47085, + "csgo": 28928, + "csi": 41750, + "csi": 28070, + "csk": 43036, + "csm": 40061, + "csn": 46329, + "cso": 43864, + "csp": 39243, + "csr": 32105, + "csr": 24598, + "csrracing": 44193, + "css": 41418, + "css": 19846, + "cst": 17016, + "csu": 35948, + "csu": 31261, + "csw": 41031, + "ct": 3381, + "ct": 1122, + "cta": 28397, + "ctar": 27842, + "ctc": 34123, + "cte": 31410, + "cted": 2910, + "ctf": 35250, + "cthulhu": 41064, + "cting": 7985, + "ction": 17578, + "ction": 1569, + "ctions": 7021, + "ctive": 9313, + "cto": 17445, + "ctor": 8108, + "ctr": 35602, + "ctr": 18481, + "cts": 6936, + "ctto": 25118, + "ctu": 20834, + "cture": 17668, + "ctv": 21213, + "ctv": 27590, + "cu": 729, + "cu": 11224, + "cuando": 40388, + "cub": 16938, + "cub": 19972, + "cuba": 11576, + "cuban": 15536, + "cube": 47753, + "cube": 11353, + "cubes": 31413, + "cubic": 48159, + "cubic": 29614, + "cubs": 9858, + "cuck": 26364, + "cuckoo": 38062, + "cucu": 16705, + "cucumber": 19787, + "cucumbers": 48065, + "cud": 42684, + "cudd": 12820, + "cuddle": 19568, + "cuddles": 24001, + "cuddling": 29696, + "cuddly": 36208, + "cudi": 48713, + "cue": 13424, + "cuer": 39506, + "cues": 35719, + "cuff": 34693, + "cuff": 22414, + "cufflinks": 43938, + "cuffs": 37221, + "cuis": 9938, + "cuisine": 10605, + "cuk": 34838, + "cul": 1877, + "cula": 35935, + "cular": 10940, + "culars": 45719, + "cule": 31066, + "cules": 18984, + "culin": 14772, + "culinary": 16466, + "cull": 21880, + "cull": 42061, + "cullen": 25973, + "culmin": 33778, + "culo": 36305, + "culprit": 41593, + "cult": 11965, + "cultiv": 16781, + "cultivate": 42983, + "cultivated": 48901, + "cultivation": 41539, + "cultur": 20780, + "cultural": 34908, + "cultural": 6753, + "culturally": 36783, + "culture": 20197, + "culture": 3673, + "cultured": 40176, + "cultures": 19552, + "culver": 42103, + "cum": 20142, + "cum": 27119, + "cumb": 10858, + "cumber": 15309, + "cumberbatch": 27541, + "cumberland": 28747, + "cumbri": 32010, + "cumbria": 17953, + "cumin": 42285, + "cumple": 47050, + "cumul": 42961, + "cumulative": 47610, + "cumulus": 46313, + "cun": 12423, + "cun": 29532, + "cunningham": 25321, + "cuomo": 25681, + "cup": 5059, + "cup": 1937, + "cupboard": 32074, + "cupcake": 17025, + "cupcakes": 12747, + "cupid": 34885, + "cuppa": 28077, + "cups": 11463, + "cur": 1092, + "cur": 33073, + "curated": 20341, + "curator": 20753, + "curb": 21931, + "curd": 38881, + "cure": 36758, + "cure": 9088, + "cured": 26248, + "cures": 38204, + "curfew": 48826, + "curi": 12640, + "curing": 44169, + "curiosity": 21583, + "curious": 9865, + "curl": 24306, + "curled": 43734, + "curling": 18543, + "curls": 24340, + "curly": 20795, + "curran": 40999, + "currant": 43501, + "curren": 6142, + "currencies": 23530, + "currency": 7853, + "current": 3653, + "currently": 3792, + "currents": 35450, + "curric": 16201, + "curriculum": 17947, + "currie": 39385, + "curry": 49285, + "curry": 8051, + "curse": 18479, + "cursed": 26408, + "cursor": 46546, + "curt": 38137, + "curtain": 17223, + "curtains": 30223, + "curti": 39925, + "curtis": 13808, + "curve": 15792, + "curved": 25789, + "curves": 22814, + "curvy": 45788, + "cus": 2736, + "cusa": 47414, + "cuse": 37950, + "cush": 43731, + "cushi": 15333, + "cushion": 20853, + "cushions": 34163, + "cussion": 16658, + "cussions": 46853, + "cust": 20900, + "custard": 26516, + "custo": 4376, + "custody": 16176, + "custom": 2662, + "custom": 4996, + "custome": 41323, + "customer": 24035, + "customer": 5102, + "customerexperience": 45167, + "customers": 5528, + "customerservice": 40611, + "customiz": 41793, + "customizable": 48253, + "customization": 48244, + "customize": 32179, + "customized": 23229, + "customs": 16880, + "cut": 10511, + "cut": 3032, + "cute": 16031, + "cute": 2242, + "cuteness": 19342, + "cuter": 27151, + "cutest": 8032, + "cuth": 44328, + "cutie": 10733, + "cuties": 40939, + "cuties": 23420, + "cutiesaturday": 41883, + "cutler": 40428, + "cutlery": 49073, + "cutout": 45016, + "cuts": 7435, + "cutt": 27338, + "cutt": 47647, + "cutter": 19719, + "cutters": 44783, + "cutting": 7266, + "cuz": 9215, + "cv": 13531, + "cv": 13947, + "cvs": 29603, + "cw": 10652, + "cw": 11065, + "cwc": 19179, + "cwgc": 48527, + "cws": 45186, + "cx": 44457, + "cx": 14283, + "cy": 1470, + "cy": 1678, + "cyber": 5830, + "cyber": 10210, + "cybercrime": 41772, + "cybermonday": 36578, + "cyberpunk": 36896, + "cybersecurity": 10581, + "cyborg": 36650, + "cycl": 9791, + "cycle": 19083, + "cycle": 5072, + "cycled": 31055, + "cycles": 14605, + "cycli": 12201, + "cycling": 26353, + "cycling": 6321, + "cyclist": 20686, + "cyclists": 20303, + "cyclo": 18122, + "cyclone": 48094, + "cyclone": 20917, + "cyclones": 34669, + "cylin": 18569, + "cylinder": 22092, + "cylinders": 48888, + "cymb": 36677, + "cymru": 24005, + "cyn": 14324, + "cynthi": 41994, + "cynthia": 23748, + "cyp": 14809, + "cypress": 25347, + "cypri": 36481, + "cyprus": 15263, + "cyril": 36028, + "cyrus": 14204, + "cystic": 46131, + "cyto": 31864, + "cz": 22898, + "cz": 22921, + "cze": 12152, + "czech": 43151, + "czech": 16141, + "cé": 36454, + "cé": 18317, + "d": 67, + "d": 323, + "da": 925, + "da": 1140, + "daa": 32642, + "daan": 44814, + "dab": 10413, + "dab": 22900, + "dac": 16222, + "dac": 27478, + "daca": 28477, + "dach": 34166, + "dachsh": 41641, + "dachshund": 42720, + "dad": 4346, + "dad": 2639, + "dada": 31325, + "daddy": 29466, + "daddy": 6546, + "dade": 23299, + "dades": 28289, + "dads": 12741, + "dae": 23358, + "dae": 15422, + "daener": 46934, + "daes": 47282, + "daesh": 35047, + "daf": 9972, + "daf": 36704, + "daffodils": 44769, + "daft": 36347, + "dag": 11434, + "dag": 25650, + "dagger": 34251, + "dah": 16976, + "dah": 11776, + "dahl": 45816, + "dahl": 22621, + "dahlia": 41768, + "dai": 13559, + "dai": 10632, + "dail": 14676, + "dailies": 21260, + "daily": 6689, + "daily": 2873, + "dailynews": 43466, + "dailys": 43160, + "dailysketch": 46738, + "daim": 40421, + "dain": 32222, + "dain": 28315, + "daipur": 47631, + "dair": 19998, + "dair": 42078, + "dairy": 25243, + "dairy": 10302, + "dairyfree": 49366, + "dais": 10502, + "daisi": 39947, + "daisies": 40654, + "daisy": 39310, + "daisy": 12865, + "dak": 6999, + "dak": 16095, + "dakar": 31137, + "dakota": 38522, + "dakota": 12358, + "dal": 2476, + "dal": 5601, + "dala": 42675, + "dalai": 41222, + "dalail": 35169, + "dalailama": 35849, + "dale": 11533, + "dale": 4677, + "dalejr": 38207, + "dales": 29031, + "daley": 28544, + "dalgo": 43614, + "dali": 36735, + "dali": 25703, + "dalit": 45432, + "dall": 43631, + "dalla": 16772, + "dallas": 27414, + "dallas": 5759, + "dallascowboys": 33016, + "dalmati": 44275, + "dalton": 21488, + "daly": 24873, + "dam": 1880, + "dam": 4926, + "damage": 6822, + "damaged": 13568, + "damages": 28842, + "damaging": 20610, + "damas": 23345, + "damascus": 25396, + "dame": 10069, + "dames": 44548, + "dami": 17783, + "damian": 43307, + "damian": 25375, + "damien": 25090, + "dammit": 31057, + "damn": 37409, + "damn": 4451, + "damned": 28428, + "damon": 48503, + "damon": 18244, + "damp": 26520, + "dams": 37680, + "dan": 2257, + "dan": 2284, + "dana": 44834, + "dana": 13777, + "danao": 38598, + "danc": 3945, + "dance": 10619, + "dance": 2724, + "danced": 32891, + "dancehall": 33300, + "dancer": 11400, + "dancers": 13153, + "dances": 24083, + "dancing": 33280, + "dancing": 6226, + "dand": 12593, + "dandelion": 38903, + "dandy": 31932, + "dane": 19330, + "danes": 47477, + "dang": 4283, + "dang": 14992, + "danger": 20083, + "danger": 11212, + "dangerous": 7350, + "dangerously": 35012, + "dangers": 23726, + "dangle": 39907, + "dani": 3001, + "dani": 17009, + "daniel": 7859, + "daniel": 4981, + "daniela": 44466, + "danielle": 30396, + "danielle": 15292, + "danielpadilla": 34702, + "daniels": 16146, + "danish": 15467, + "dank": 31849, + "dann": 11951, + "danny": 14950, + "danny": 7621, + "dano": 29703, + "dans": 16241, + "dant": 48097, + "dant": 28237, + "dante": 21911, + "danube": 44594, + "dany": 47816, + "dao": 36099, + "dap": 12149, + "dap": 38034, + "daph": 24591, + "daphne": 31687, + "dapl": 34478, + "dapp": 46857, + "dapper": 26071, + "daq": 25381, + "dar": 1377, + "dar": 6242, + "dara": 17064, + "darby": 34366, + "darcy": 32916, + "dare": 14833, + "dare": 9863, + "daredevil": 28849, + "dares": 42973, + "dareto": 46794, + "dari": 16292, + "dari": 14552, + "daria": 45622, + "daries": 18184, + "daring": 28166, + "dario": 33918, + "darius": 32606, + "darje": 49089, + "dark": 5724, + "dark": 3144, + "darker": 18737, + "darkest": 25898, + "darkness": 10521, + "darling": 13048, + "darlings": 39961, + "darlington": 34565, + "darn": 26059, + "darrell": 33522, + "darren": 20263, + "darren": 12275, + "darry": 29200, + "darryl": 35359, + "darshan": 34564, + "dart": 14001, + "dart": 19841, + "darth": 41304, + "darth": 23164, + "dartmoor": 31477, + "dartmouth": 29667, + "darts": 15246, + "darwin": 43013, + "darwin": 20926, + "daryl": 45607, + "daryl": 24532, + "das": 9940, + "das": 7359, + "dash": 13858, + "dash": 10206, + "dashboard": 27679, + "dashi": 12876, + "dashing": 33825, + "dat": 1717, + "dat": 9445, + "data": 14876, + "data": 2281, + "datab": 11941, + "database": 14678, + "databases": 48384, + "datac": 27329, + "datacenter": 40133, + "datasci": 14496, + "datascience": 15748, + "dataviz": 28138, + "date": 34300, + "date": 1524, + "dated": 13564, + "dates": 7228, + "dating": 8534, + "dation": 15311, + "datlantic": 34270, + "dato": 36075, + "dats": 48674, + "dau": 3162, + "dau": 33828, + "daugh": 42523, + "daughter": 3944, + "daughters": 13585, + "daun": 29470, + "dav": 3700, + "dav": 46488, + "davao": 31502, + "dave": 10089, + "dave": 5077, + "daven": 28350, + "davenport": 34624, + "davey": 33391, + "davi": 1732, + "david": 4640, + "david": 2259, + "davidbowie": 44448, + "davido": 35989, + "davids": 46695, + "davidson": 13166, + "davies": 13120, + "davin": 43187, + "davis": 24426, + "davis": 5536, + "davison": 43725, + "davos": 31887, + "davy": 41565, + "daw": 5971, + "daw": 24404, + "dawg": 18660, + "dawgs": 26431, + "dawn": 30590, + "dawn": 7689, + "dawson": 18611, + "dax": 29458, + "day": 1405, + "day": 575, + "daya": 38165, + "daybreak": 33862, + "daycare": 36363, + "daydream": 41587, + "dayin": 20332, + "daylight": 20809, + "dayo": 29856, + "dayo": 46605, + "dayof": 16272, + "dayofthe": 38043, + "days": 1161, + "daysof": 12379, + "daysofcode": 36537, + "daysto": 29886, + "daystogo": 42198, + "dayswild": 42052, + "daytime": 22830, + "dayton": 35729, + "dayton": 20262, + "daytona": 16335, + "dayweekend": 44526, + "dayz": 35949, + "daz": 15449, + "daz": 43844, + "daze": 33591, + "dazz": 17149, + "dazzle": 41164, + "dazzling": 28821, + "db": 19100, + "db": 8128, + "dbacks": 31175, + "dbs": 40558, + "dbz": 49226, + "dc": 5074, + "dc": 2743, + "dca": 49107, + "dcc": 33747, + "dccomics": 17610, + "dcfc": 35526, + "dci": 35336, + "dcs": 42878, + "dcu": 42647, + "dd": 1353, + "dd": 3766, + "dda": 35202, + "ddad": 39049, + "dday": 32689, + "dday": 26243, + "ddc": 48513, + "ddd": 24183, + "dddd": 35362, + "dden": 5013, + "dder": 9300, + "dders": 24827, + "ddi": 44450, + "ddin": 17175, + "dding": 48101, + "dding": 8974, + "ddings": 49106, + "ddington": 29238, + "ddle": 17633, + "ddle": 8357, + "ddled": 38392, + "ddles": 33901, + "ddleston": 25647, + "ddling": 30981, + "ddlovato": 28244, + "ddos": 46463, + "ddr": 26027, + "dds": 48334, + "ddu": 43836, + "ddy": 14981, + "ddy": 7876, + "de": 561, + "de": 654, + "dea": 18477, + "deacon": 29155, + "dead": 3906, + "dead": 2747, + "deadliest": 40811, + "deadline": 47209, + "deadline": 8458, + "deadlines": 44959, + "deadly": 10756, + "deadpool": 21471, + "deaf": 28229, + "deaf": 18358, + "deal": 7249, + "deal": 2696, + "dealer": 15218, + "dealers": 21697, + "dealership": 32096, + "dealing": 13138, + "deals": 4469, + "dealt": 30101, + "dean": 13807, + "dean": 5828, + "deandre": 43635, + "deans": 46852, + "dear": 15696, + "dear": 3817, + "dearest": 24880, + "dearly": 31880, + "deas": 34715, + "death": 7163, + "death": 2767, + "deaths": 12253, + "deau": 12399, + "deaux": 19883, + "deb": 2987, + "deb": 25687, + "debat": 32082, + "debate": 5196, + "debates": 19239, + "debating": 23472, + "debbie": 47186, + "debbie": 16735, + "debit": 32410, + "debor": 16738, + "deborah": 40997, + "deborah": 22150, + "debra": 33233, + "debris": 19208, + "debt": 8932, + "debts": 38770, + "debu": 9790, + "debun": 33123, + "debut": 42608, + "debut": 4085, + "debuted": 25215, + "debuting": 34817, + "debuts": 17044, + "dec": 3063, + "dec": 4628, + "deca": 33428, + "decad": 29914, + "decade": 11099, + "decadent": 41716, + "decades": 10488, + "decal": 26678, + "decals": 37606, + "decan": 40677, + "decat": 35334, + "decath": 47455, + "decatur": 38540, + "decay": 22703, + "dece": 3534, + "deceased": 30035, + "december": 3864, + "decent": 10698, + "decentr": 28960, + "decentralized": 38485, + "decep": 33529, + "deception": 33046, + "deci": 2262, + "decide": 8447, + "decided": 4939, + "decides": 17269, + "deciding": 22513, + "decision": 5575, + "decisions": 9903, + "decisive": 28690, + "deck": 24885, + "deck": 6943, + "decked": 39096, + "decker": 21449, + "decks": 23968, + "decl": 7091, + "decla": 10739, + "declan": 42341, + "declar": 18040, + "declaration": 19714, + "declare": 19856, + "declared": 13845, + "declares": 23641, + "declaring": 33273, + "decline": 15084, + "declined": 28911, + "declines": 40478, + "declining": 29221, + "deco": 26412, + "deco": 16422, + "decor": 5148, + "decor": 6928, + "decorate": 23651, + "decorated": 15917, + "decorating": 16968, + "decoration": 16029, + "decorations": 19158, + "decorative": 19289, + "decre": 12284, + "decrease": 24703, + "decreased": 33913, + "decreasing": 43763, + "decree": 43327, + "ded": 16744, + "ded": 1241, + "dedic": 4701, + "dedicate": 27610, + "dedicated": 6770, + "dedication": 10188, + "dedly": 36204, + "deduc": 22799, + "dee": 5268, + "dee": 6705, + "deed": 30260, + "deeds": 24516, + "deejay": 48304, + "deejay": 44511, + "deemed": 28102, + "deen": 26456, + "deen": 12912, + "deep": 5462, + "deep": 3383, + "deepak": 45528, + "deeper": 15224, + "deepest": 22245, + "deephouse": 35684, + "deepi": 19371, + "deepika": 34120, + "deepikap": 29903, + "deepikapadukone": 30646, + "deeplear": 22181, + "deeplearning": 24362, + "deeply": 11449, + "deer": 19454, + "deer": 8700, + "deere": 32901, + "dees": 12547, + "deets": 35537, + "def": 2044, + "def": 11649, + "defam": 35670, + "defamation": 42741, + "default": 21650, + "defe": 4148, + "defeat": 8477, + "defeated": 8927, + "defeating": 22594, + "defeats": 16317, + "defect": 44013, + "defects": 37485, + "defen": 3619, + "defence": 30307, + "defence": 9659, + "defend": 21970, + "defend": 11397, + "defended": 27161, + "defender": 10618, + "defenders": 20063, + "defending": 13098, + "defends": 20134, + "defense": 45875, + "defense": 6021, + "defenseman": 43714, + "defenses": 49198, + "defensive": 10824, + "defi": 17244, + "defiance": 36186, + "defiant": 47597, + "defibrill": 47684, + "defic": 18022, + "defici": 23387, + "deficiency": 30685, + "deficit": 20156, + "defin": 3188, + "define": 14919, + "defined": 15278, + "defines": 28218, + "defining": 20504, + "definite": 40793, + "definitely": 4824, + "definition": 11405, + "definitive": 25298, + "defl": 31467, + "deforestation": 41330, + "defstar": 36427, + "defy": 39148, + "defying": 38496, + "deg": 38498, + "degra": 28939, + "degradation": 44468, + "degre": 4653, + "degree": 7119, + "degrees": 8000, + "deh": 35582, + "dei": 33833, + "dei": 23279, + "deir": 42948, + "deity": 42574, + "deja": 46902, + "dek": 23901, + "dekalb": 37775, + "del": 1233, + "del": 2003, + "dela": 37986, + "delaney": 31528, + "delav": 23706, + "delavin": 40477, + "delavin": 40776, + "delavinkisses": 40631, + "delaware": 17547, + "delay": 12955, + "delay": 10934, + "delayed": 14567, + "delaying": 43781, + "delays": 11232, + "dele": 7922, + "dele": 33431, + "delec": 38615, + "delectable": 45500, + "deleg": 8046, + "delegate": 27259, + "delegates": 14623, + "delegation": 14632, + "delete": 19204, + "deleted": 16588, + "deleting": 41857, + "delft": 42749, + "delgado": 49182, + "delhi": 26723, + "delhi": 5717, + "deli": 1932, + "deli": 18601, + "delia": 33193, + "deliber": 18316, + "deliberate": 38271, + "deliberately": 35163, + "delic": 13366, + "delicacy": 49181, + "delicate": 18768, + "delici": 19993, + "delicious": 3959, + "deliciously": 39589, + "deliciousness": 42819, + "delight": 46165, + "delight": 13073, + "delighted": 5943, + "delightful": 15513, + "delights": 25330, + "deline": 18797, + "delines": 13562, + "delish": 25093, + "deliver": 19561, + "deliver": 7396, + "delivered": 7278, + "deliveries": 29336, + "delivering": 9943, + "delivers": 11753, + "delivery": 5619, + "dell": 24381, + "dell": 10242, + "della": 22986, + "delle": 35963, + "deloit": 29428, + "deloitte": 38667, + "dels": 48636, + "delta": 32250, + "delta": 8768, + "delu": 18779, + "delusional": 48059, + "delux": 13709, + "deluxe": 14056, + "delve": 46008, + "dely": 15040, + "dem": 3251, + "dem": 7825, + "dema": 40268, + "dema": 45046, + "deman": 48366, + "demand": 13072, + "demand": 5650, + "demanded": 33699, + "demanding": 17099, + "demands": 14241, + "demar": 46566, + "demarcus": 47873, + "demb": 35930, + "demdebate": 43973, + "deme": 25143, + "demean": 37376, + "demen": 12604, + "dementi": 46028, + "dementia": 14047, + "demetri": 39553, + "demi": 32879, + "demi": 14480, + "demise": 28756, + "demo": 2930, + "demo": 7380, + "democr": 3573, + "democracy": 7758, + "democrat": 15431, + "democratic": 9149, + "democrats": 8865, + "demographic": 31308, + "demol": 19382, + "demolished": 26537, + "demolition": 22237, + "demon": 5635, + "demon": 12085, + "demonetisation": 41338, + "demonic": 46920, + "demons": 18388, + "demonstr": 8579, + "demonstrate": 22231, + "demonstrated": 29477, + "demonstrates": 24806, + "demonstrating": 22107, + "demonstration": 16722, + "demonstrations": 33964, + "demonstrators": 46450, + "demos": 19304, + "demp": 22490, + "dempsey": 30188, + "dems": 10989, + "demsin": 42664, + "demsinphilly": 43091, + "den": 1177, + "den": 1181, + "dena": 32431, + "denali": 48076, + "dence": 3370, + "dency": 11659, + "dend": 37447, + "dends": 43985, + "dene": 45128, + "dened": 19571, + "deng": 43098, + "deng": 41788, + "dengue": 41932, + "denham": 39180, + "deni": 21995, + "denial": 25716, + "denied": 15780, + "denies": 19565, + "denim": 13606, + "denis": 47630, + "denis": 18750, + "denise": 45900, + "denise": 20899, + "denmark": 13268, + "dennis": 32738, + "dennis": 10534, + "denny": 26808, + "denomin": 41016, + "dens": 16533, + "dense": 19353, + "density": 22431, + "dent": 3593, + "dent": 1258, + "dental": 24635, + "dental": 8382, + "dentally": 10346, + "dented": 21923, + "denti": 4418, + "dential": 5459, + "dentist": 17816, + "dentistry": 25754, + "dently": 28817, + "denton": 23567, + "dents": 1517, + "denver": 27847, + "denver": 8569, + "deny": 18679, + "denying": 32771, + "denzel": 42503, + "deo": 26406, + "deo": 12121, + "deodor": 47639, + "deol": 41902, + "deon": 31466, + "deon": 16079, + "dep": 6079, + "dep": 24370, + "depar": 10794, + "depart": 5343, + "depart": 30649, + "departed": 32541, + "departing": 26902, + "department": 5744, + "departments": 29523, + "departs": 38998, + "departure": 17850, + "depe": 36118, + "depend": 13894, + "depend": 27371, + "dependence": 40243, + "dependent": 23280, + "depending": 23673, + "depends": 20497, + "depic": 11307, + "depicted": 34637, + "depicting": 24970, + "depiction": 31071, + "depicts": 29340, + "deple": 38504, + "deplo": 9356, + "deplor": 39232, + "deploy": 26944, + "deployed": 20009, + "deploying": 42212, + "deployment": 20183, + "depo": 14276, + "depor": 36110, + "deport": 23389, + "deportation": 36617, + "deported": 39320, + "deportes": 47878, + "depos": 21266, + "deposit": 16775, + "deposits": 30740, + "depot": 12589, + "depp": 24941, + "depre": 7107, + "depress": 38869, + "depressed": 23269, + "depressing": 29235, + "depression": 10023, + "depri": 28587, + "depriv": 45809, + "deprivation": 47810, + "deprived": 39140, + "dept": 9201, + "depth": 10350, + "depths": 28855, + "depu": 6912, + "deputies": 24914, + "deputy": 7932, + "der": 839, + "der": 801, + "dera": 20696, + "derail": 48502, + "derby": 13904, + "derby": 7177, + "derbyshire": 22147, + "derdale": 21513, + "dere": 5701, + "dere": 44194, + "dered": 3776, + "derek": 22461, + "derek": 11205, + "derel": 46728, + "derer": 11289, + "derers": 20882, + "deri": 34573, + "derick": 33908, + "dering": 6076, + "deriv": 33458, + "derived": 26461, + "derland": 35488, + "derman": 29740, + "dermatology": 48051, + "dern": 30086, + "dero": 37203, + "dero": 34026, + "derrick": 21798, + "derry": 45777, + "derry": 20535, + "ders": 37307, + "ders": 1923, + "derson": 12677, + "dery": 17172, + "des": 6797, + "des": 1437, + "desai": 35316, + "desc": 13866, + "descen": 32318, + "descend": 26004, + "descend": 46241, + "descendants": 36323, + "descending": 36620, + "descent": 19375, + "desch": 49209, + "descri": 4637, + "describe": 10967, + "described": 14671, + "describes": 13678, + "describing": 24239, + "descrip": 41832, + "description": 13951, + "descriptions": 40653, + "desde": 42218, + "dese": 27195, + "deser": 3659, + "desert": 45776, + "desert": 7301, + "deserted": 41560, + "deserve": 7043, + "deserved": 10061, + "deserves": 9079, + "deserving": 26615, + "desh": 25320, + "desh": 7448, + "deshi": 42769, + "desi": 6772, + "desi": 26635, + "desig": 1250, + "design": 8359, + "design": 1681, + "designated": 24119, + "designation": 41155, + "designed": 4486, + "designer": 35640, + "designer": 5728, + "designers": 12720, + "designing": 13467, + "designs": 6747, + "designthinking": 32450, + "desirable": 32368, + "desire": 11858, + "desired": 28631, + "desires": 27598, + "desk": 11937, + "desk": 6550, + "desks": 41014, + "desktop": 14345, + "desmond": 27821, + "desol": 41258, + "desp": 3642, + "despair": 28097, + "desper": 10144, + "desperate": 15072, + "desperately": 21993, + "despic": 32442, + "despicable": 37158, + "despite": 5325, + "dess": 7096, + "dess": 10001, + "dessert": 9753, + "desserts": 22948, + "desses": 43913, + "dest": 6540, + "dest": 4549, + "destin": 4934, + "destination": 32191, + "destination": 9179, + "destinations": 16981, + "destined": 28525, + "destiny": 39875, + "destiny": 10867, + "destro": 8287, + "destroy": 8308, + "destroy": 11930, + "destroyed": 9965, + "destroyer": 25291, + "destroying": 19613, + "destroys": 27634, + "destruc": 22945, + "destruction": 14281, + "destructive": 29591, + "det": 28966, + "det": 15366, + "deta": 1914, + "detached": 26252, + "detail": 7657, + "detailed": 12609, + "detailing": 23163, + "details": 2353, + "detained": 20260, + "dete": 5606, + "detec": 17991, + "detect": 22744, + "detected": 26988, + "detecting": 41290, + "detection": 16220, + "detective": 13672, + "detectives": 27994, + "detector": 27689, + "detectors": 45063, + "detention": 16908, + "deter": 10742, + "deter": 47458, + "detergent": 46726, + "deterior": 28512, + "determin": 8325, + "determination": 17410, + "determine": 16768, + "determined": 14371, + "determines": 42192, + "determining": 39884, + "deth": 38375, + "deto": 39710, + "deton": 39335, + "detour": 31211, + "detox": 22459, + "detri": 47951, + "detro": 6210, + "detroit": 19404, + "detroit": 7073, + "detta": 45438, + "dette": 35750, + "deu": 21457, + "deuce": 45332, + "deus": 37625, + "deut": 14970, + "deutsch": 30389, + "deutsche": 32760, + "deutschland": 36878, + "deux": 47089, + "dev": 2797, + "dev": 3670, + "deva": 45179, + "devan": 37072, + "devast": 12913, + "devastated": 29865, + "devastating": 19280, + "devastation": 42452, + "devel": 1820, + "develop": 1966, + "develop": 7708, + "developed": 8763, + "developer": 10929, + "developers": 13248, + "developing": 8131, + "development": 2855, + "developmental": 29347, + "developments": 17393, + "develops": 29895, + "deven": 45537, + "devgn": 29871, + "devi": 12926, + "devi": 20717, + "deviant": 25593, + "deviantart": 26046, + "device": 8163, + "devices": 9067, + "devil": 8894, + "devil": 8043, + "deville": 34329, + "devils": 11683, + "devin": 31193, + "devin": 20996, + "devine": 33019, + "devlin": 48040, + "devo": 11861, + "devo": 43444, + "devon": 16205, + "devon": 10046, + "devops": 21504, + "devos": 40646, + "devote": 37777, + "devoted": 24561, + "devotees": 39759, + "devotion": 25821, + "devotional": 35456, + "devs": 27374, + "dew": 31952, + "dew": 16358, + "dewey": 40399, + "dex": 10030, + "dex": 13790, + "dexpo": 42502, + "dexter": 45049, + "dexter": 22781, + "dey": 11829, + "dez": 23190, + "dez": 8122, + "df": 12908, + "df": 10468, + "dfc": 41903, + "dfs": 32880, + "dfw": 20439, + "dg": 2394, + "dg": 9742, + "dgate": 41684, + "dge": 4016, + "dge": 1360, + "dged": 11830, + "dgeon": 45655, + "dgers": 8733, + "dges": 5432, + "dging": 9565, + "dh": 6669, + "dh": 9960, + "dha": 11629, + "dha": 27377, + "dhabi": 22349, + "dhaka": 32877, + "dham": 29635, + "dham": 30838, + "dhan": 12542, + "dhan": 28569, + "dhanush": 26162, + "dhanush": 36200, + "dhanushkraja": 29266, + "dhar": 12397, + "dharma": 30536, + "dhary": 28706, + "dhawan": 44699, + "dhe": 29706, + "dheim": 44280, + "dhi": 31553, + "dhi": 26166, + "dho": 37834, + "dhoni": 25698, + "dhru": 40257, + "dhry": 39960, + "dhs": 26849, + "dhu": 32387, + "di": 570, + "di": 1618, + "dia": 7351, + "dia": 3357, + "diab": 15954, + "diabe": 19167, + "diabete": 43826, + "diabetes": 10319, + "diabetic": 30230, + "diablo": 23931, + "diag": 6851, + "diagno": 7736, + "diagnose": 44429, + "diagnosed": 16979, + "diagnosis": 15715, + "diagnostic": 26351, + "diagnostics": 37723, + "diagram": 22697, + "dial": 18416, + "dial": 11381, + "dialo": 30709, + "dialog": 48945, + "dialogue": 11288, + "dialogues": 40330, + "dialysis": 44798, + "diam": 4347, + "diameter": 27189, + "diamon": 8873, + "diamond": 18535, + "diamond": 6235, + "diamonds": 12687, + "dian": 16021, + "dian": 4998, + "diana": 12803, + "diane": 15855, + "dianne": 42299, + "dians": 21041, + "diaper": 34382, + "diapers": 39659, + "diar": 25932, + "diaries": 15541, + "diary": 10380, + "dias": 22137, + "dias": 29354, + "diaspora": 28390, + "diaz": 17688, + "dic": 1404, + "dic": 6717, + "dicap": 30023, + "dicaprio": 30755, + "dice": 14406, + "dick": 14413, + "dick": 9554, + "dickens": 33421, + "dict": 45360, + "dict": 15159, + "dictat": 26156, + "dictator": 27399, + "dictatorship": 37989, + "dictionary": 19699, + "did": 1861, + "did": 1335, + "diddy": 33527, + "didi": 34396, + "didier": 45614, + "didn": 2376, + "didnt": 13057, + "dido": 31725, + "didyou": 12295, + "didyouknow": 12506, + "die": 3150, + "die": 2082, + "diec": 27729, + "diecast": 37936, + "died": 3622, + "diego": 30940, + "diego": 6306, + "diem": 45571, + "dience": 33686, + "dient": 27231, + "dier": 29702, + "dier": 16394, + "dies": 20104, + "dies": 1862, + "diesel": 46312, + "diesel": 10591, + "diest": 45739, + "diet": 21295, + "diet": 6582, + "dietary": 29009, + "dietrich": 47005, + "diets": 35173, + "dif": 18656, + "dif": 48731, + "diff": 44073, + "diff": 20331, + "diffe": 1967, + "differ": 34620, + "differen": 14903, + "difference": 4731, + "differences": 14003, + "different": 2731, + "differenti": 21729, + "differential": 34027, + "differentiate": 49032, + "differently": 18325, + "diffic": 6140, + "difficult": 7405, + "difficulties": 23468, + "difficulty": 25245, + "diffu": 31603, + "diffuser": 49400, + "dig": 1831, + "dig": 9887, + "dige": 17820, + "digest": 20413, + "digestion": 40533, + "digestive": 32304, + "digg": 43240, + "digger": 35919, + "diggin": 48466, + "digging": 14971, + "digi": 15627, + "digi": 39361, + "digimon": 44181, + "digit": 14899, + "digit": 27472, + "digital": 4704, + "digital": 2794, + "digitalart": 16987, + "digitalhealth": 32190, + "digitalindia": 46630, + "digitally": 27543, + "digitalmarketing": 15299, + "digitaltransformation": 20047, + "digiti": 25935, + "digits": 31710, + "digni": 45532, + "dignit": 39497, + "dignity": 17744, + "digo": 35701, + "digs": 26877, + "dih": 43089, + "dii": 32755, + "dijk": 44444, + "dik": 38854, + "dik": 37747, + "dike": 42683, + "dil": 7643, + "dil": 17942, + "dile": 25428, + "dilemma": 29787, + "dilig": 30664, + "dill": 12318, + "dill": 27206, + "dillon": 21056, + "dilu": 45242, + "dim": 19576, + "dim": 17523, + "dime": 24443, + "dimen": 10935, + "dimension": 20479, + "dimensional": 25252, + "dimensions": 25086, + "diment": 43500, + "dimes": 44888, + "dimini": 37459, + "dimit": 22250, + "dimitri": 48840, + "dimp": 38853, + "din": 1462, + "din": 5673, + "dina": 36815, + "dinah": 30903, + "dine": 20951, + "dine": 12989, + "diner": 16963, + "dinesh": 48341, + "ding": 7545, + "ding": 796, + "dinger": 45580, + "dingh": 48064, + "dings": 5473, + "dington": 24804, + "dinho": 47370, + "dini": 20196, + "dining": 8658, + "dinner": 27548, + "dinner": 2571, + "dinners": 33570, + "dino": 9692, + "dino": 14077, + "dinosa": 18955, + "dinosaur": 15095, + "dinosaurs": 20387, + "dio": 3779, + "dio": 1521, + "dioce": 20763, + "diocese": 27091, + "dion": 42899, + "dion": 16250, + "dior": 23655, + "dios": 37563, + "dious": 27417, + "dioxide": 38102, + "dip": 19918, + "dip": 11343, + "dipl": 8490, + "diplo": 38115, + "diplom": 11169, + "diploma": 21251, + "diplomacy": 23798, + "diplomat": 32828, + "diplomatic": 23782, + "diplomats": 44126, + "dipped": 30610, + "dipper": 49317, + "dipping": 33544, + "dips": 37522, + "dir": 4251, + "dir": 8478, + "dire": 38355, + "dire": 25664, + "direc": 1534, + "direct": 43224, + "direct": 6016, + "directed": 8392, + "directing": 21817, + "direction": 15923, + "direction": 5407, + "directional": 38687, + "directioner": 48042, + "directioners": 22055, + "directions": 16440, + "directive": 40630, + "directly": 9701, + "director": 20337, + "director": 2681, + "directorial": 45327, + "directors": 11940, + "directory": 25272, + "directs": 34349, + "directv": 48652, + "dirk": 28171, + "dirt": 31415, + "dirt": 11795, + "dirty": 20127, + "dirty": 7615, + "dis": 1518, + "dis": 6112, + "disa": 3882, + "disab": 47380, + "disabilities": 17350, + "disability": 48986, + "disability": 13261, + "disabled": 13613, + "disadvantaged": 40577, + "disagree": 23199, + "disapp": 5384, + "disappear": 21148, + "disappear": 25173, + "disappearance": 35929, + "disappeared": 23139, + "disappearing": 35819, + "disappears": 44406, + "disappo": 7605, + "disappoint": 25446, + "disappointed": 13794, + "disappointing": 21941, + "disappointment": 23884, + "disappoints": 48545, + "disappro": 48276, + "disar": 42971, + "disaster": 9072, + "disasters": 26976, + "disastrous": 35790, + "disc": 1472, + "disc": 10712, + "discar": 40532, + "discarded": 45197, + "discer": 49140, + "dischar": 22671, + "discharge": 32485, + "disci": 9559, + "discip": 38951, + "discipl": 10467, + "disciples": 39366, + "disciplinary": 20232, + "discipline": 18903, + "disciplines": 42032, + "discla": 40248, + "disclaimer": 46465, + "disclo": 17481, + "disclose": 46379, + "disclosed": 30905, + "disclosure": 26502, + "disco": 2475, + "disco": 11964, + "discography": 47545, + "discomfort": 48054, + "discord": 23582, + "discoun": 18515, + "discount": 7638, + "discounted": 20993, + "discounts": 18186, + "discoura": 45850, + "discourse": 29441, + "discover": 10539, + "discover": 4834, + "discovered": 6986, + "discoveries": 29308, + "discovering": 17967, + "discovers": 29719, + "discovery": 40491, + "discovery": 8027, + "discre": 20616, + "discrimin": 11721, + "discrimination": 14775, + "discs": 29270, + "discu": 1984, + "discus": 41828, + "discuss": 4312, + "discussed": 11300, + "discusses": 8116, + "discussing": 5900, + "discussion": 5060, + "discussions": 13806, + "dise": 4262, + "disease": 5336, + "diseases": 12035, + "disen": 46468, + "disgrace": 29877, + "disgraceful": 44146, + "disgu": 9793, + "disguise": 27803, + "disguised": 37149, + "disgusted": 41977, + "disgusting": 16218, + "dish": 11039, + "dish": 4531, + "disha": 42498, + "dishes": 11412, + "dishon": 30777, + "dishu": 44728, + "dishwasher": 40524, + "disin": 19484, + "disinfe": 48050, + "disintegr": 49275, + "disk": 17970, + "dislike": 30796, + "dism": 30836, + "dism": 38821, + "dismant": 36557, + "dismiss": 43287, + "dismissal": 42068, + "dismissed": 30087, + "dismisses": 45238, + "disney": 6729, + "disney": 4696, + "disneyland": 39481, + "disneyland": 13661, + "disneyworld": 28469, + "diso": 26305, + "disobe": 42841, + "dison": 19310, + "disorder": 12635, + "disorders": 17114, + "disp": 11073, + "dispar": 24633, + "disparities": 45122, + "dispat": 28652, + "dispatch": 26306, + "dispen": 19077, + "dispenser": 40116, + "disper": 34499, + "displa": 9326, + "displac": 17718, + "displaced": 22817, + "displacement": 37931, + "display": 4456, + "displayed": 18967, + "displaying": 26468, + "displays": 15648, + "dispo": 13651, + "dispon": 38872, + "disponible": 46130, + "dispos": 45177, + "disposable": 37275, + "disposal": 28231, + "dispro": 32927, + "dispropor": 40354, + "disproportion": 45492, + "disregard": 43869, + "disrespect": 34055, + "disrespectful": 41723, + "disru": 13763, + "disrup": 14641, + "disrupt": 25214, + "disrupted": 46674, + "disrupting": 42419, + "disruption": 19635, + "disruptive": 31554, + "diss": 10766, + "diss": 35688, + "dissec": 43879, + "dissemin": 40463, + "dissent": 45154, + "disser": 25560, + "dissertation": 29448, + "dissi": 25088, + "dissol": 27398, + "dissuper": 33461, + "dist": 5479, + "dist": 12116, + "distance": 7964, + "distances": 37078, + "distant": 18949, + "distill": 41586, + "distilled": 49179, + "distillery": 22200, + "distin": 11892, + "distinct": 25056, + "distinction": 28183, + "distinctive": 25486, + "distingui": 15053, + "distinguish": 45418, + "distinguished": 16513, + "distor": 23781, + "distortion": 43690, + "distr": 11885, + "distract": 39309, + "distracted": 24049, + "distraction": 32039, + "distress": 26866, + "distressed": 37515, + "distri": 5987, + "distribu": 6138, + "distribute": 32313, + "distributed": 16419, + "distributing": 35216, + "distribution": 10484, + "distributor": 28354, + "distributors": 44240, + "distric": 3208, + "district": 46683, + "district": 3506, + "districts": 17565, + "distur": 11732, + "disturb": 33018, + "disturb": 39449, + "disturbance": 42416, + "disturbed": 29967, + "disturbing": 21476, + "disupdates": 45667, + "dit": 5752, + "dit": 2524, + "dita": 47965, + "ditch": 43715, + "ditch": 19291, + "dited": 40392, + "diti": 2363, + "dition": 16452, + "dition": 3015, + "ditional": 4322, + "ditions": 4503, + "dito": 43705, + "dits": 49374, + "dity": 16436, + "dium": 2903, + "div": 5293, + "div": 14869, + "diva": 13605, + "divas": 23534, + "dive": 26042, + "dive": 9058, + "diver": 13119, + "diver": 22094, + "divergence": 48735, + "divergent": 36132, + "divers": 30241, + "divers": 27038, + "diverse": 11464, + "diversi": 24475, + "diversion": 38457, + "diversity": 35634, + "diversity": 6257, + "diverted": 41049, + "dives": 13893, + "divi": 8375, + "divid": 31337, + "divide": 18842, + "divided": 18689, + "dividend": 32067, + "dividends": 45146, + "dividing": 45605, + "divin": 21838, + "divine": 46919, + "divine": 10976, + "diving": 9886, + "divinity": 39754, + "divisi": 39196, + "division": 5378, + "divisional": 40912, + "divisions": 33715, + "divor": 13543, + "divorce": 17060, + "divorced": 39437, + "divya": 47767, + "diwali": 18218, + "dix": 45838, + "dix": 27620, + "dixie": 24484, + "dixit": 28279, + "dixon": 16086, + "diy": 28472, + "diy": 7845, + "diya": 36459, + "diz": 32740, + "dized": 36232, + "dizz": 40239, + "dizzy": 35464, + "dj": 3761, + "dj": 3723, + "djan": 35338, + "django": 46498, + "dji": 35284, + "dji": 28379, + "djing": 36113, + "djo": 19432, + "djoker": 42721, + "djokernole": 42830, + "djokovic": 27944, + "djs": 18117, + "dk": 20702, + "dk": 16196, + "dl": 12558, + "dl": 9373, + "dlc": 19079, + "dle": 11057, + "dle": 3287, + "dled": 23494, + "dler": 40279, + "dles": 7890, + "dless": 14997, + "dley": 12808, + "dling": 18221, + "dly": 3069, + "dm": 19070, + "dm": 4667, + "dma": 42903, + "dman": 18826, + "dmc": 28991, + "dmit": 31607, + "dmitry": 48326, + "dms": 19955, + "dmv": 27508, + "dmx": 45255, + "dn": 11552, + "dn": 7459, + "dna": 8790, + "dnb": 35422, + "dnc": 20237, + "dnd": 11678, + "dnr": 37051, + "dns": 39245, + "dnt": 26795, + "do": 639, + "do": 818, + "doa": 48332, + "dob": 29640, + "doba": 35605, + "dobbs": 43006, + "dobson": 46888, + "doc": 3009, + "doc": 7251, + "doch": 25101, + "dock": 17311, + "dock": 8997, + "docked": 46784, + "docker": 31152, + "docking": 40845, + "docks": 24091, + "docs": 15157, + "doctor": 7872, + "doctor": 5547, + "doctoral": 23649, + "doctorate": 39134, + "doctors": 9705, + "doctorwho": 12996, + "doctr": 28497, + "doctrine": 35612, + "docu": 4433, + "document": 29293, + "document": 15121, + "documentaries": 44209, + "documentary": 7881, + "documentation": 31560, + "documented": 22310, + "documenting": 37876, + "documents": 14105, + "dod": 13847, + "dod": 30187, + "dodd": 36748, + "dodge": 31263, + "dodge": 12093, + "dodgeball": 43244, + "dodger": 31641, + "dodgers": 12422, + "dodgy": 37727, + "doe": 13296, + "does": 2397, + "does": 1897, + "doesn": 2503, + "doesnt": 17937, + "dof": 8277, + "doff": 20193, + "dofficial": 42516, + "dog": 4326, + "dog": 1929, + "dogcelebration": 41819, + "dogday": 27475, + "doge": 42187, + "dogg": 20749, + "doggie": 32237, + "doggo": 42155, + "doggy": 26359, + "doglo": 40733, + "dogre": 40030, + "dogrescue": 44158, + "dogs": 42182, + "dogs": 3255, + "dogsoftwitter": 19415, + "doh": 23581, + "doha": 20908, + "doherty": 31774, + "doi": 36361, + "doin": 15412, + "doing": 37408, + "doing": 1960, + "doit": 32272, + "doit": 28109, + "doj": 25700, + "dojo": 35901, + "dok": 40547, + "dok": 41034, + "doka": 46528, + "dol": 2287, + "dol": 19170, + "dola": 38005, + "dolan": 27200, + "dolby": 42414, + "dolce": 30033, + "dolce": 30661, + "dole": 41040, + "doll": 27031, + "doll": 9286, + "dollar": 35092, + "dollar": 7474, + "dollars": 10669, + "dolls": 15090, + "dolly": 43281, + "dolly": 23821, + "dolom": 37137, + "dolores": 40741, + "dolph": 8900, + "dolph": 22257, + "dolphin": 42963, + "dolphin": 16464, + "dolphins": 14002, + "dom": 2164, + "dom": 1919, + "domain": 15492, + "domaine": 48744, + "domains": 36358, + "dome": 8515, + "dome": 9827, + "domen": 37584, + "domest": 21936, + "domestic": 28189, + "domestic": 9043, + "domin": 4361, + "dominance": 30546, + "dominant": 20565, + "dominate": 21431, + "dominated": 23048, + "dominates": 34043, + "dominating": 29303, + "domination": 30919, + "domingo": 24882, + "dominic": 39007, + "dominic": 19095, + "dominican": 22934, + "dominion": 27155, + "domino": 30752, + "dominos": 39770, + "domo": 44293, + "doms": 30126, + "don": 1067, + "don": 847, + "dona": 26789, + "donal": 42375, + "donald": 5990, + "donald": 4335, + "donaldson": 37783, + "donaldtrump": 6652, + "donat": 36384, + "donate": 6429, + "donated": 8705, + "donates": 26960, + "donating": 12621, + "donation": 7924, + "donations": 9928, + "doncaster": 38008, + "doncaster": 25352, + "doncasterisgreat": 47333, + "done": 5136, + "done": 1700, + "donegal": 24172, + "donesia": 41281, + "donet": 33724, + "donetsk": 33999, + "dong": 26242, + "dong": 31478, + "dongha": 28365, + "donghae": 28945, + "donia": 24014, + "donkey": 21415, + "donkeys": 44644, + "donna": 9158, + "donne": 30897, + "donnein": 38308, + "donneinarte": 40193, + "donnell": 35118, + "donnelly": 39070, + "donnie": 47058, + "donnie": 30609, + "donny": 37291, + "donny": 32887, + "dono": 14840, + "donor": 18013, + "donors": 17887, + "donovan": 21499, + "dons": 22127, + "dont": 8094, + "dont": 4632, + "donut": 18471, + "donuts": 13970, + "doo": 4543, + "doo": 11643, + "doodle": 9388, + "doodled": 41030, + "doodles": 22156, + "doodling": 37548, + "dooley": 47609, + "doom": 23263, + "doom": 14344, + "doomed": 33251, + "doomsday": 41791, + "doon": 36612, + "doop": 33886, + "door": 7188, + "door": 2489, + "doors": 4228, + "doorstep": 19533, + "doorway": 46575, + "dop": 42381, + "dop": 31722, + "dope": 42587, + "dope": 10094, + "doping": 30285, + "dopp": 21774, + "doppelg": 45216, + "doppler": 42540, + "dor": 2766, + "dor": 8695, + "dora": 18104, + "dorado": 32350, + "dorchester": 32656, + "dore": 39423, + "dores": 34323, + "dorf": 17296, + "dori": 49270, + "doria": 43186, + "dorian": 44016, + "doris": 24285, + "dork": 36206, + "dorm": 24263, + "doro": 15498, + "doro": 37389, + "dorothy": 20805, + "dors": 31240, + "dorset": 42109, + "dorset": 16047, + "dorsey": 41607, + "dortmund": 24290, + "dory": 36135, + "dos": 44258, + "dos": 5474, + "dose": 11497, + "doses": 37873, + "dossier": 46042, + "dost": 44222, + "dot": 7473, + "dot": 7004, + "dota": 23085, + "dotcom": 12443, + "dote": 31202, + "dothis": 47864, + "dotnet": 43124, + "dotorg": 46587, + "dots": 19019, + "dotted": 47950, + "dou": 1756, + "dou": 23608, + "doub": 19631, + "double": 13013, + "double": 3200, + "doubled": 24948, + "doubleheader": 34668, + "doubles": 12539, + "doubling": 36850, + "doubt": 37071, + "doubt": 8671, + "doubts": 30894, + "douche": 44292, + "doug": 20271, + "doug": 10758, + "dough": 15785, + "dough": 14983, + "doughnut": 32555, + "doughnuts": 31124, + "dougie": 46317, + "dougla": 9140, + "douglas": 10065, + "douglass": 45692, + "doun": 44785, + "dov": 38856, + "dova": 26551, + "dove": 27511, + "dove": 18281, + "dover": 43019, + "dover": 14683, + "doves": 47067, + "dow": 8022, + "dow": 10688, + "dowell": 27344, + "down": 1833, + "down": 1136, + "downe": 46501, + "downed": 35814, + "downer": 42522, + "downers": 43739, + "downey": 29429, + "downfall": 48702, + "downhill": 27387, + "downing": 28140, + "download": 35076, + "download": 3794, + "downloadable": 49105, + "downloaded": 22961, + "downloading": 30519, + "downloads": 26481, + "downpour": 39034, + "downpours": 40160, + "downs": 10706, + "downside": 41937, + "downstairs": 28174, + "downstream": 43822, + "downtime": 41964, + "downton": 45023, + "downton": 42668, + "downtown": 18230, + "downtown": 5061, + "downward": 37430, + "dowski": 43556, + "dox": 44786, + "dox": 14510, + "doyle": 17728, + "doyou": 27256, + "doz": 31106, + "dozen": 16401, + "dozens": 17883, + "dp": 23820, + "dp": 6465, + "dprint": 46644, + "dprinting": 16194, + "dprk": 47920, + "dps": 34288, + "dq": 28741, + "dr": 1084, + "dr": 1701, + "dra": 1114, + "dra": 7402, + "drac": 20168, + "dracing": 41253, + "dracula": 25405, + "draf": 37426, + "draft": 30624, + "draft": 5198, + "drafted": 19129, + "drafting": 33528, + "drafts": 29194, + "drag": 8452, + "drag": 12463, + "dragged": 27884, + "dragging": 37069, + "dragon": 9187, + "dragon": 5471, + "dragonball": 40959, + "dragoncon": 47802, + "dragonfly": 32824, + "dragons": 10203, + "dragrace": 40762, + "drags": 45368, + "drain": 23347, + "drain": 19467, + "drainage": 25953, + "drained": 44630, + "drains": 43638, + "drainthe": 47337, + "drake": 32504, + "drake": 8958, + "dral": 7503, + "dram": 6937, + "dram": 32170, + "drama": 5055, + "dramas": 33467, + "dramati": 43512, + "dramatic": 11240, + "dramatically": 24495, + "drank": 21712, + "draped": 49113, + "drastic": 43159, + "drastically": 35478, + "drau": 18621, + "draw": 17675, + "draw": 4001, + "drawer": 23219, + "drawers": 38975, + "drawing": 36996, + "drawing": 3610, + "drawings": 13397, + "drawn": 8893, + "draws": 12043, + "dray": 25562, + "drayton": 49044, + "drc": 21434, + "dre": 960, + "dre": 14584, + "dread": 17412, + "dread": 31403, + "dreaded": 47227, + "dreadful": 35846, + "dreality": 48367, + "dream": 4595, + "dream": 2984, + "dreambig": 46495, + "dreamcast": 47226, + "dreamed": 27984, + "dreamer": 25692, + "dreamers": 27194, + "dreaming": 11662, + "dreamliner": 49143, + "dreams": 4405, + "dreamt": 43743, + "dreamteam": 40090, + "dreamy": 23517, + "dred": 10903, + "dredge": 48783, + "dren": 29068, + "dren": 47309, + "drenched": 46378, + "dres": 48852, + "dres": 44697, + "dresden": 34836, + "dress": 12622, + "dress": 2595, + "dressage": 36144, + "dressed": 6559, + "dresser": 26346, + "dresses": 8184, + "dressing": 6348, + "drew": 18792, + "drew": 5281, + "drex": 33985, + "drey": 48271, + "dri": 1203, + "dri": 28833, + "drian": 36870, + "dribb": 42153, + "dric": 23448, + "dridge": 22956, + "drie": 40170, + "dried": 16037, + "drier": 39877, + "dries": 33857, + "drif": 33585, + "drift": 18194, + "drifting": 30276, + "drill": 11626, + "drilled": 46338, + "drilling": 18634, + "drills": 24378, + "drin": 3375, + "drin": 47133, + "drink": 14131, + "drink": 3979, + "drinking": 5778, + "drinklocal": 45998, + "drinks": 6732, + "drip": 24050, + "dripping": 38787, + "dris": 35804, + "drive": 11402, + "drive": 2620, + "driven": 9314, + "driver": 27563, + "driver": 4383, + "driverless": 46769, + "drivers": 7384, + "drives": 11441, + "driveway": 26273, + "driving": 37800, + "driving": 4161, + "drizzle": 28240, + "drm": 39674, + "dro": 1494, + "dro": 12442, + "drogba": 49199, + "droid": 38016, + "drome": 9157, + "dron": 43898, + "dron": 23360, + "drone": 33557, + "drone": 9397, + "drones": 14006, + "droo": 30715, + "drool": 41554, + "drooling": 44360, + "drop": 16407, + "drop": 3387, + "dropbox": 47216, + "dropped": 6792, + "dropping": 8339, + "drops": 6437, + "dros": 47033, + "drou": 38558, + "drought": 13935, + "drove": 13753, + "drow": 21159, + "drown": 28571, + "drowned": 34005, + "drowning": 24618, + "drs": 21257, + "dru": 2275, + "dru": 49048, + "drug": 20601, + "drug": 5600, + "drugs": 8021, + "druid": 40297, + "drum": 13353, + "drum": 8698, + "drummer": 13618, + "drummers": 46191, + "drumming": 35480, + "drummond": 42213, + "drums": 11690, + "drun": 15488, + "drunk": 37398, + "drunk": 8232, + "drunken": 28196, + "drupal": 46481, + "drush": 43009, + "drwho": 48342, + "dry": 13544, + "dry": 4501, + "dryer": 24425, + "drying": 23203, + "ds": 3361, + "ds": 646, + "dsa": 47607, + "dsb": 47168, + "dsb": 14257, + "dsburg": 47237, + "dsc": 37240, + "dsd": 45383, + "dsley": 40740, + "dslr": 33740, + "dsm": 39502, + "dson": 40310, + "dsp": 45291, + "dss": 41580, + "dstv": 35027, + "dt": 13104, + "dt": 7427, + "dthe": 13863, + "dtla": 31885, + "dtm": 42407, + "dts": 46233, + "du": 691, + "du": 3686, + "dua": 25244, + "dual": 39739, + "dual": 5347, + "duane": 38946, + "dub": 14526, + "dub": 13144, + "duba": 5485, + "dubai": 32599, + "dubai": 5985, + "dubbed": 27740, + "dublin": 20707, + "dublin": 6145, + "dubnation": 47329, + "dubois": 48046, + "dubrov": 46709, + "dubrovnik": 48724, + "dubs": 27013, + "dubstep": 38303, + "dubu": 43257, + "duc": 979, + "duc": 36446, + "ducati": 28570, + "ducation": 17197, + "duce": 3660, + "duchess": 21713, + "duck": 12708, + "duck": 6910, + "ducks": 11202, + "duct": 26829, + "dude": 48087, + "dude": 5710, + "dudes": 14449, + "dudley": 27324, + "due": 2887, + "duel": 27143, + "dues": 37646, + "duet": 25457, + "duf": 38713, + "duff": 38071, + "duff": 21934, + "duffy": 23599, + "dug": 22743, + "dug": 21000, + "dugg": 40523, + "duggan": 46169, + "dugout": 36831, + "duh": 26716, + "dui": 29693, + "duk": 14160, + "duke": 18402, + "duke": 7732, + "dukes": 27914, + "dul": 6738, + "dulce": 44872, + "dulil": 32565, + "dulkar": 47980, + "dull": 19433, + "dulu": 28865, + "duluth": 32109, + "dulwich": 47343, + "dum": 13400, + "dum": 11564, + "dumb": 15901, + "dumb": 12464, + "dumbass": 38980, + "dummies": 40899, + "dummy": 34246, + "dump": 12655, + "dump": 17146, + "dumped": 23768, + "dumping": 31707, + "dumplings": 35495, + "dumps": 45804, + "dumpster": 45467, + "dun": 2616, + "dun": 18284, + "dunbar": 41453, + "duncan": 31084, + "duncan": 13502, + "dundal": 38185, + "dundas": 39300, + "dundee": 18619, + "dune": 32833, + "dune": 28208, + "dunedin": 40121, + "dunes": 23526, + "dung": 33712, + "dungeon": 28812, + "dungeon": 22931, + "dungeons": 42572, + "dungeonsand": 34970, + "dungeonsanddragons": 35497, + "dunham": 42501, + "duni": 43454, + "dunk": 17222, + "dunkin": 48022, + "dunkin": 36415, + "dunkirk": 46928, + "dunks": 48977, + "dunlop": 34753, + "dunn": 19185, + "dunne": 38538, + "dunno": 24502, + "duo": 8696, + "dup": 36805, + "dup": 10445, + "duper": 44850, + "duplex": 41186, + "duplic": 28992, + "dupont": 35994, + "dur": 4355, + "dur": 23230, + "dura": 28173, + "dura": 47382, + "durability": 43671, + "durable": 22285, + "duran": 28185, + "durango": 44443, + "durant": 24861, + "duras": 27518, + "duration": 31663, + "durban": 24474, + "dure": 19108, + "durga": 38456, + "durham": 26765, + "durham": 14335, + "during": 1590, + "dus": 9931, + "dusa": 28546, + "dusk": 19708, + "dust": 29723, + "dust": 8349, + "dusted": 38274, + "duster": 46280, + "dustin": 42423, + "dustin": 21235, + "dusting": 41756, + "dusty": 22029, + "dut": 32625, + "dutch": 22277, + "dutch": 7991, + "duter": 21624, + "duterte": 22371, + "duties": 19603, + "dutt": 30081, + "dutton": 42771, + "duty": 6458, + "duval": 42459, + "duvet": 48006, + "dux": 28562, + "dv": 4288, + "dv": 26265, + "dvd": 7170, + "dvds": 36655, + "dvn": 29811, + "dvr": 29210, + "dw": 8455, + "dw": 19997, + "dwar": 13487, + "dwarf": 22643, + "dwayne": 31395, + "dwell": 27549, + "dwell": 18755, + "dwelling": 37098, + "dwight": 22473, + "dwp": 46976, + "dwts": 30220, + "dwyer": 43878, + "dx": 22717, + "dx": 15679, + "dy": 1444, + "dy": 907, + "dyce": 48325, + "dye": 37159, + "dye": 15997, + "dyed": 24906, + "dyer": 29495, + "dyes": 39874, + "dying": 5115, + "dyk": 12142, + "dyke": 32632, + "dylan": 21004, + "dylan": 9900, + "dyn": 44289, + "dyn": 30669, + "dynam": 5735, + "dynamic": 10057, + "dynamics": 14329, + "dynamite": 29003, + "dynamo": 28281, + "dynasty": 14593, + "dyne": 42756, + "dyou": 11484, + "dyour": 22525, + "dys": 11022, + "dys": 38384, + "dysfunction": 36865, + "dysfunctional": 40757, + "dysle": 33681, + "dyslexia": 43199, + "dyson": 34475, + "dyssey": 17435, + "dystop": 28276, + "dystopian": 38915, + "dz": 24421, + "dz": 22913, + "dé": 25466, + "dü": 46948, + "dÃŃ": 46988, + "e": 68, + "e": 324, + "ea": 2150, + "ea": 8100, + "eable": 20693, + "each": 31442, + "each": 2416, + "eachother": 40792, + "ead": 42556, + "ead": 45523, + "eae": 27446, + "eag": 3743, + "eager": 21551, + "eagerly": 30094, + "eagle": 20207, + "eagle": 7517, + "eagles": 6920, + "eal": 48872, + "ealing": 40484, + "eames": 49072, + "eamon": 45954, + "ean": 13327, + "ear": 1055, + "ear": 8373, + "earbuds": 47807, + "eared": 9127, + "earl": 30573, + "earl": 14235, + "earle": 40292, + "earlier": 4297, + "earliest": 22097, + "early": 15840, + "early": 2090, + "earn": 33977, + "earn": 8465, + "earned": 8898, + "earnest": 45422, + "earning": 14550, + "earnings": 15912, + "earns": 16760, + "earp": 35296, + "earphones": 44905, + "earring": 28664, + "earrings": 9136, + "ears": 9861, + "eart": 7086, + "earth": 5184, + "earth": 3475, + "earthand": 34229, + "earthandclouds": 34480, + "earthday": 19481, + "earthquake": 10060, + "earthquakes": 32895, + "earthy": 47139, + "earts": 38824, + "eas": 5740, + "ease": 13574, + "easier": 8817, + "easiest": 26314, + "easily": 8197, + "easing": 44825, + "easport": 42251, + "east": 5022, + "east": 2602, + "eastbound": 28827, + "eastbourne": 38455, + "eastenders": 23545, + "easter": 14783, + "easter": 4811, + "eastern": 34522, + "eastern": 6311, + "eastman": 48280, + "easton": 29619, + "eastside": 42650, + "eastwood": 28270, + "easy": 18308, + "easy": 3176, + "eat": 5418, + "eat": 3384, + "eaten": 16750, + "eater": 24060, + "eaters": 37645, + "eatery": 46559, + "eating": 4371, + "eatlocal": 42868, + "eaton": 28462, + "eats": 13188, + "eau": 17608, + "eazy": 36536, + "eb": 12283, + "eb": 8677, + "eba": 40889, + "ebay": 34412, + "ebay": 4099, + "eber": 34020, + "ebo": 46635, + "ebola": 15864, + "ebon": 22013, + "ebony": 30651, + "ebook": 13122, + "ebooks": 25774, + "ec": 747, + "ec": 10879, + "eca": 18465, + "ecar": 34500, + "ecb": 26205, + "ecc": 33128, + "eccc": 47401, + "eccentric": 43228, + "eccle": 27494, + "ece": 2163, + "eces": 5905, + "ecg": 45983, + "ech": 15797, + "ech": 31147, + "echel": 41233, + "echo": 17366, + "echo": 13989, + "echoes": 32564, + "eci": 31936, + "eck": 25866, + "eck": 15969, + "ecker": 39661, + "ecker": 40890, + "ecla": 47806, + "eclec": 25114, + "eclectic": 28382, + "eclip": 30841, + "eclipse": 11505, + "eclub": 38983, + "eco": 5106, + "eco": 10077, + "ecofriendly": 43412, + "ecol": 22706, + "ecological": 25127, + "ecology": 18578, + "ecommerce": 15529, + "econ": 26755, + "econ": 21158, + "econom": 2768, + "economic": 36649, + "economic": 5259, + "economical": 48782, + "economically": 39406, + "economics": 12625, + "economies": 27136, + "economist": 18836, + "economists": 43701, + "economy": 5644, + "ecor": 28962, + "ecosystem": 15788, + "ecosystems": 28725, + "ecoun": 27924, + "ecr": 48572, + "ecraft": 11439, + "ecs": 23485, + "ecstasy": 47286, + "ecstatic": 36244, + "ect": 25168, + "ecu": 13087, + "ecu": 32919, + "ecuador": 19813, + "ecz": 43530, + "ed": 843, + "ed": 538, + "eda": 10804, + "edad": 44724, + "eday": 39258, + "edc": 21245, + "edchat": 14702, + "edd": 35431, + "eddi": 42930, + "eddie": 22748, + "eddie": 9517, + "eddy": 25959, + "ede": 29632, + "eded": 19555, + "edel": 20460, + "edelman": 48139, + "eden": 23621, + "eden": 13741, + "eder": 16249, + "edes": 36247, + "edfringe": 27402, + "edg": 35955, + "edgar": 33543, + "edgar": 17914, + "edge": 16914, + "edge": 5461, + "edged": 39188, + "edges": 20938, + "edgy": 35393, + "edi": 8750, + "edi": 27148, + "edible": 19795, + "edic": 25184, + "edics": 30641, + "edin": 6524, + "edinburgh": 27574, + "edinburgh": 8068, + "eding": 5742, + "edison": 25846, + "edit": 8239, + "edit": 8013, + "edited": 13945, + "edith": 28597, + "editing": 10178, + "edition": 3062, + "editions": 21664, + "editor": 7661, + "editorial": 12325, + "editors": 19486, + "edits": 24945, + "edm": 37843, + "edm": 13539, + "edmon": 11275, + "edmond": 41581, + "edmonds": 46520, + "edmonton": 37311, + "edmonton": 15058, + "edmun": 36561, + "edmund": 27567, + "edna": 39002, + "edo": 29145, + "edo": 18096, + "edon": 41467, + "edor": 30184, + "edou": 47678, + "edp": 46066, + "eds": 1941, + "edsheeran": 30386, + "edt": 15071, + "edtech": 41825, + "edtech": 15262, + "edu": 11757, + "edu": 11799, + "eduardo": 30604, + "educ": 2200, + "educate": 17563, + "educated": 21447, + "education": 22358, + "education": 2806, + "educational": 10400, + "educator": 19875, + "educators": 15420, + "edwar": 27586, + "edward": 26184, + "edward": 7450, + "edwards": 12627, + "edwin": 48718, + "edwin": 22471, + "edy": 17072, + "edy": 4144, + "ee": 2644, + "ee": 4708, + "eed": 17513, + "eee": 24632, + "eee": 9361, + "eeee": 11696, + "eeee": 17570, + "eeeee": 26938, + "eeeeee": 41407, + "eek": 46591, + "eel": 27462, + "eels": 44416, + "eem": 27236, + "een": 47490, + "een": 21230, + "eer": 35409, + "eer": 31846, + "eera": 36664, + "eerie": 33846, + "ees": 40308, + "eet": 48935, + "eez": 39033, + "ef": 1490, + "ef": 1829, + "efa": 16999, + "eface": 48804, + "efan": 33556, + "efc": 22065, + "efcc": 46087, + "efer": 26199, + "eff": 20548, + "eff": 21715, + "effe": 2808, + "effec": 3943, + "effect": 5436, + "effective": 6837, + "effectively": 17516, + "effectiveness": 26847, + "effects": 7331, + "effic": 36004, + "efficacy": 39937, + "effici": 6670, + "efficiency": 11823, + "efficient": 11334, + "efficiently": 32915, + "effor": 6356, + "effort": 40078, + "effort": 6255, + "effortless": 41639, + "effortlessly": 42320, + "efforts": 6847, + "efish": 35813, + "efl": 27172, + "efron": 48111, + "efs": 7389, + "eg": 8053, + "eg": 14599, + "ega": 41193, + "egan": 42943, + "eger": 46704, + "eger": 22767, + "egg": 13778, + "egg": 5911, + "eggplant": 34906, + "eggs": 7099, + "ego": 34712, + "ego": 14250, + "egos": 43992, + "egre": 27044, + "egret": 42002, + "egy": 5224, + "egyp": 10250, + "egypt": 7267, + "egyptian": 12428, + "eh": 9277, + "eh": 9135, + "eha": 48563, + "ehealth": 48617, + "ehr": 45271, + "ehs": 44648, + "ei": 4006, + "ei": 18264, + "eic": 40251, + "eid": 28038, + "eid": 13979, + "eidmubarak": 46275, + "eiffel": 29720, + "eigh": 13468, + "eight": 7910, + "eighteen": 49316, + "eighth": 21237, + "eighty": 47449, + "eil": 29457, + "eileen": 31468, + "ein": 29944, + "ein": 24524, + "eindhoven": 47172, + "eing": 7702, + "einstein": 20587, + "eira": 47708, + "eis": 13802, + "eisen": 25273, + "eisenhower": 35562, + "either": 6036, + "ej": 19887, + "ej": 25009, + "ejec": 29771, + "ek": 4212, + "ek": 2092, + "el": 544, + "el": 832, + "ela": 11284, + "ela": 3787, + "elab": 38866, + "elabor": 26034, + "elaborate": 33855, + "elaine": 22523, + "elan": 17763, + "elan": 18399, + "eland": 24930, + "eland": 6275, + "elas": 41078, + "elast": 27479, + "elastic": 30282, + "elba": 48598, + "elbow": 21965, + "eld": 5684, + "elder": 11791, + "elder": 14416, + "elderly": 15455, + "elders": 28617, + "eldest": 33503, + "elding": 28223, + "elds": 13466, + "ele": 2084, + "ele": 9766, + "eleague": 36577, + "eleanor": 18604, + "elearning": 29969, + "elec": 1564, + "elec": 38768, + "elect": 15336, + "elected": 8828, + "election": 19312, + "election": 4247, + "electionday": 40540, + "elections": 6949, + "elector": 16465, + "electoral": 19544, + "electr": 3654, + "electra": 48959, + "electri": 23927, + "electric": 19547, + "electric": 5031, + "electrical": 12176, + "electrician": 46422, + "electricity": 10950, + "electrifying": 48843, + "electro": 11648, + "electro": 23244, + "electromagnetic": 46530, + "electron": 33396, + "electronic": 33865, + "electronic": 9273, + "electronica": 43119, + "electronics": 13081, + "eled": 20357, + "elee": 44112, + "eleg": 8075, + "elegance": 19146, + "elegant": 11124, + "elek": 34559, + "elem": 25406, + "element": 14909, + "elementary": 8143, + "elements": 10925, + "elen": 30654, + "elen": 39164, + "elena": 19421, + "eleng": 48180, + "eleph": 7554, + "elephant": 10299, + "elephants": 16871, + "eler": 24646, + "eless": 15244, + "eless": 30837, + "elets": 19400, + "elev": 7921, + "elevate": 26736, + "elevated": 23967, + "elevation": 23826, + "elevator": 19021, + "eleven": 31617, + "eleven": 17795, + "elf": 45961, + "elf": 11924, + "elfie": 39955, + "elg": 28790, + "elgin": 31868, + "eli": 1018, + "eli": 6292, + "elia": 10956, + "elian": 42508, + "elias": 47274, + "elias": 29902, + "elic": 34743, + "elic": 13492, + "elie": 38677, + "elie": 26501, + "elier": 14634, + "elife": 37429, + "elife": 12719, + "eligibility": 34937, + "eligible": 16978, + "elijah": 26065, + "elike": 48913, + "elim": 9296, + "elimin": 11386, + "eliminate": 19655, + "eliminated": 29075, + "eliminating": 36619, + "elimination": 24176, + "elin": 25353, + "elin": 13458, + "eline": 46199, + "eline": 7153, + "eling": 9990, + "elio": 47943, + "elion": 30682, + "elions": 44159, + "eliot": 33326, + "elis": 23411, + "elis": 48021, + "elisa": 25610, + "elisa": 44051, + "elisabeth": 33127, + "elise": 27124, + "elit": 40882, + "elite": 32277, + "elite": 6553, + "elited": 43943, + "elitedangerous": 47138, + "elites": 35975, + "elius": 35623, + "elive": 49338, + "elive": 23505, + "elives": 49174, + "elix": 32926, + "elixir": 42887, + "eliz": 42844, + "eliza": 6132, + "eliza": 29992, + "elizabeth": 22397, + "elizabeth": 7026, + "elk": 34013, + "elk": 21896, + "ell": 826, + "ell": 812, + "ella": 20692, + "ella": 2957, + "elland": 43326, + "ellar": 38443, + "ellas": 37053, + "elle": 12818, + "elle": 4765, + "elled": 13146, + "ellen": 14007, + "ellen": 12312, + "ellenshow": 34812, + "eller": 20927, + "eller": 4465, + "ellers": 19010, + "elles": 24431, + "elli": 3367, + "elli": 6673, + "ellic": 38905, + "ellie": 16769, + "ellier": 44054, + "ellin": 40374, + "elling": 2220, + "ellington": 34477, + "ellini": 43256, + "elliot": 20761, + "elliott": 44456, + "elliott": 13788, + "ellip": 44816, + "ellis": 11553, + "ellison": 32295, + "ello": 2512, + "ellor": 14594, + "ells": 2433, + "ellu": 35560, + "elly": 8041, + "elly": 20355, + "elm": 25199, + "elm": 22082, + "elman": 33622, + "elmer": 45958, + "elmo": 32150, + "elo": 6170, + "elo": 13490, + "elon": 26381, + "elon": 20406, + "elondon": 47377, + "elong": 44363, + "elonmusk": 37076, + "elope": 23367, + "eloqu": 37795, + "elos": 44733, + "elot": 43490, + "elove": 43319, + "elove": 19165, + "elover": 21732, + "elovers": 33946, + "els": 35958, + "els": 1645, + "elsa": 22050, + "else": 18857, + "else": 3344, + "elsewhere": 22906, + "elson": 19624, + "elt": 18692, + "elton": 20758, + "elu": 14208, + "elusive": 28903, + "elves": 29111, + "elvi": 47008, + "elvis": 47359, + "elvis": 14498, + "elxn": 37726, + "ely": 12189, + "ely": 1273, + "elyn": 29691, + "elyn": 18126, + "em": 908, + "em": 2270, + "ema": 7002, + "ema": 11131, + "emabiggest": 23101, + "emabiggestfans": 29587, + "email": 33537, + "email": 4462, + "emailed": 40470, + "emailmarketing": 40188, + "emails": 12871, + "eman": 24416, + "eman": 36868, + "emancip": 42996, + "emanuel": 35232, + "emb": 3692, + "embar": 8266, + "embaras": 48019, + "embark": 33953, + "embarra": 11382, + "embarrass": 27183, + "embarrassed": 28217, + "embarrassing": 19653, + "embarrassment": 41346, + "embassy": 13598, + "embe": 46041, + "embed": 19703, + "embedded": 22046, + "embelli": 32144, + "embellished": 46992, + "ember": 47049, + "emblem": 21163, + "embo": 23065, + "embr": 35267, + "embrac": 16928, + "embrace": 12118, + "embraced": 35739, + "embraces": 38404, + "embracing": 22196, + "embro": 12550, + "embroi": 18667, + "embroide": 21530, + "embroidered": 22381, + "embroidery": 20823, + "emc": 20897, + "emc": 31602, + "emcee": 42038, + "eme": 22910, + "eme": 21548, + "emea": 40352, + "emed": 11028, + "emen": 22033, + "ement": 40841, + "ement": 2057, + "ements": 11058, + "emer": 3132, + "emer": 25727, + "emerald": 46878, + "emerald": 16980, + "emerge": 22182, + "emerged": 26425, + "emergen": 24096, + "emergence": 39867, + "emergencies": 35759, + "emergency": 44038, + "emergency": 5897, + "emerges": 30801, + "emerging": 38174, + "emerging": 11113, + "emeritus": 35333, + "emerson": 24147, + "emery": 32678, + "emi": 44327, + "emi": 18525, + "emil": 26794, + "emil": 40624, + "emile": 43926, + "emili": 20709, + "emilia": 34238, + "emilio": 39722, + "emily": 14545, + "emily": 7640, + "emin": 17227, + "emin": 23995, + "eminem": 22129, + "eminent": 33779, + "eming": 40398, + "emir": 13337, + "emir": 47613, + "emirates": 47244, + "emirates": 17867, + "emission": 27761, + "emissions": 14172, + "emit": 49043, + "emma": 18177, + "emma": 7445, + "emmanuel": 48045, + "emmanuel": 20411, + "emmett": 45779, + "emmy": 35625, + "emmy": 17089, + "emmys": 21875, + "emo": 3738, + "emo": 19381, + "emoji": 16327, + "emojis": 27870, + "emon": 34406, + "emor": 45034, + "emory": 44274, + "emotion": 17464, + "emotional": 7357, + "emotionally": 24088, + "emotions": 12904, + "emp": 3831, + "emp": 41004, + "empathy": 22420, + "emper": 12522, + "emperor": 13828, + "empha": 16237, + "emphasi": 47176, + "emphasis": 29588, + "empire": 26212, + "empire": 7614, + "empires": 46510, + "emplo": 3409, + "employ": 37290, + "employ": 39626, + "employe": 5037, + "employed": 26567, + "employee": 36631, + "employee": 9560, + "employees": 7377, + "employer": 21296, + "employers": 17647, + "employment": 10959, + "empor": 27386, + "emporium": 48541, + "empower": 13612, + "empower": 17230, + "empowered": 29087, + "empowering": 20086, + "empowerment": 15747, + "empowers": 46206, + "empress": 26656, + "empty": 41203, + "empty": 7893, + "emra": 39259, + "ems": 2858, + "emt": 46360, + "emu": 48149, + "emu": 29296, + "emul": 23272, + "emy": 31076, + "en": 524, + "en": 576, + "ena": 3452, + "enab": 17308, + "enable": 15642, + "enabled": 23666, + "enables": 23417, + "enabling": 23590, + "enam": 41486, + "enamel": 22746, + "enary": 13132, + "enas": 34536, + "enation": 20860, + "enberg": 15658, + "enburg": 28430, + "enc": 33169, + "enca": 37774, + "encan": 30345, + "encapsul": 40874, + "ence": 6495, + "ence": 954, + "enced": 6549, + "ences": 3777, + "enchan": 17290, + "enchanted": 28258, + "enchanting": 32531, + "enchil": 47396, + "enci": 32207, + "encia": 30068, + "encies": 18729, + "encing": 10326, + "enclosed": 43243, + "enclosure": 37419, + "encom": 44026, + "encore": 20549, + "encoun": 17309, + "encounter": 13164, + "encountered": 32492, + "encounters": 25399, + "encoura": 6169, + "encourage": 12090, + "encouraged": 20299, + "encouragement": 24959, + "encourages": 23848, + "encouraging": 15875, + "encro": 45822, + "encry": 28600, + "encryp": 42928, + "encrypted": 48710, + "encryption": 31423, + "ency": 3484, + "encyclo": 32104, + "encyclopedia": 38376, + "end": 945, + "end": 806, + "enda": 6735, + "endale": 20290, + "endange": 13990, + "endangered": 14931, + "ende": 11373, + "ende": 40306, + "endeav": 18134, + "endeavor": 40502, + "endeavors": 44394, + "endeavour": 38035, + "ended": 2622, + "endemic": 41241, + "endent": 16265, + "ender": 48106, + "ender": 12383, + "enders": 7418, + "endez": 43850, + "endgame": 23042, + "endi": 31359, + "ending": 2695, + "endings": 36516, + "endish": 38841, + "endless": 12688, + "endlessly": 45145, + "endment": 45894, + "endo": 13476, + "endo": 15830, + "endocr": 36486, + "endof": 40786, + "endome": 46996, + "endon": 48018, + "endor": 8092, + "endorf": 37249, + "endorse": 28819, + "endorsed": 24307, + "endorsement": 21205, + "endorses": 34603, + "endorsing": 46779, + "endow": 45895, + "endra": 22321, + "ends": 1339, + "endthe": 46256, + "endu": 26032, + "endur": 19557, + "endurance": 21027, + "endure": 32419, + "enduring": 30851, + "enduro": 47042, + "ene": 3297, + "ene": 6049, + "ened": 2494, + "eneed": 45137, + "enegger": 33235, + "enei": 48906, + "enemies": 15824, + "enemy": 10310, + "enen": 45113, + "ener": 2244, + "ener": 13600, + "energ": 39451, + "energetic": 24197, + "energi": 23044, + "energies": 42374, + "energized": 48635, + "energy": 14974, + "energy": 2650, + "energye": 32271, + "energyefficiency": 40586, + "eners": 48208, + "enes": 42066, + "eness": 11806, + "enet": 46336, + "enew": 29672, + "enews": 13442, + "eney": 20706, + "enez": 33110, + "enf": 38167, + "enfield": 27808, + "enfor": 10592, + "enforce": 40224, + "enforced": 44597, + "enforcement": 12460, + "eng": 1035, + "eng": 6730, + "enga": 22297, + "engag": 6793, + "engage": 11089, + "engaged": 11475, + "engagement": 7281, + "engaging": 13060, + "enge": 26279, + "enge": 2742, + "engel": 38265, + "engen": 48286, + "enger": 6618, + "engers": 7533, + "engine": 3355, + "engine": 5857, + "engineer": 40151, + "engineer": 8517, + "engineered": 26580, + "engineering": 5273, + "engineers": 11494, + "engines": 14487, + "england": 20904, + "england": 3595, + "english": 15942, + "english": 3469, + "engra": 17560, + "engraved": 29421, + "engraving": 33309, + "engul": 43655, + "engv": 28401, + "enh": 7449, + "enhall": 48781, + "enham": 24592, + "enhan": 26827, + "enhance": 13993, + "enhanced": 16070, + "enhancement": 35601, + "enhances": 38259, + "enhancing": 25986, + "eni": 4395, + "eni": 17538, + "enic": 46780, + "enic": 28292, + "enig": 19754, + "enig": 48730, + "enight": 32848, + "enight": 20640, + "enigma": 34998, + "ening": 1133, + "enium": 34380, + "enix": 25720, + "enjo": 1498, + "enjoy": 12981, + "enjoy": 2218, + "enjoyable": 17444, + "enjoyed": 5045, + "enjoying": 3603, + "enjoyment": 34905, + "enjoys": 17024, + "enka": 43942, + "enko": 25312, + "enlar": 38136, + "enligh": 21364, + "enlighten": 28200, + "enlightened": 44032, + "enlightening": 44005, + "enlightenment": 29255, + "enlisted": 43555, + "enly": 43023, + "enn": 43563, + "enna": 8095, + "enne": 21176, + "enne": 11518, + "ennedy": 46266, + "ennes": 43613, + "enni": 7049, + "ennial": 14220, + "ennis": 48923, + "ennis": 26309, + "eno": 9429, + "eno": 12843, + "enoch": 47917, + "enor": 13955, + "enormous": 20129, + "enos": 44759, + "enote": 44955, + "enough": 2744, + "enow": 26876, + "enqu": 28417, + "enqui": 22810, + "enquire": 46658, + "enquiries": 31901, + "enquiry": 45141, + "enri": 18915, + "enrich": 20058, + "enrich": 45504, + "enriched": 45166, + "enrichment": 32903, + "enrique": 25489, + "enrol": 44279, + "enroll": 23739, + "enroll": 30366, + "enrolled": 36853, + "enrollment": 24875, + "enroute": 40548, + "ens": 41799, + "ens": 1323, + "ense": 12657, + "ense": 27658, + "ensemble": 14843, + "ensis": 32842, + "ensla": 37535, + "enslaved": 48675, + "ensure": 7492, + "ensures": 29707, + "ensuring": 19403, + "ent": 724, + "ent": 621, + "enta": 17681, + "ental": 32342, + "ental": 6168, + "entary": 9833, + "entation": 37412, + "ente": 17433, + "ente": 9935, + "ented": 3800, + "entennial": 43088, + "enter": 2963, + "enter": 3819, + "entered": 10679, + "entering": 12580, + "enterpri": 7339, + "enterprise": 9220, + "enterprises": 21219, + "enters": 15287, + "entertain": 5566, + "entertain": 23510, + "entertained": 30631, + "entertainer": 28674, + "entertaining": 13897, + "entertainment": 6166, + "entes": 24213, + "enthr": 36202, + "enthusi": 9631, + "enthusiasm": 20525, + "enthusiast": 27153, + "enthusiastic": 22068, + "enthusiasts": 27514, + "enti": 1938, + "ential": 5194, + "entially": 37695, + "entic": 10340, + "entine": 49212, + "enting": 20526, + "entire": 4709, + "entirely": 13911, + "entirety": 43242, + "entit": 15209, + "entities": 38134, + "entitled": 18680, + "entity": 28455, + "ently": 2922, + "ento": 21917, + "ento": 8762, + "entom": 31676, + "entourage": 47893, + "entr": 7129, + "entrance": 9129, + "entrata": 27304, + "entre": 34188, + "entre": 19600, + "entren": 46959, + "entrepre": 4583, + "entreprene": 4789, + "entrepreneu": 26784, + "entrepreneur": 12119, + "entrepreneur": 8033, + "entrepreneurial": 28261, + "entrepreneurs": 11054, + "entrepreneurship": 12858, + "entries": 13766, + "entry": 5362, + "ents": 870, + "entu": 6650, + "enty": 5657, + "enu": 23430, + "env": 32280, + "env": 39207, + "envel": 20052, + "envelope": 27358, + "envir": 3512, + "enviro": 46200, + "environ": 3599, + "environment": 33039, + "environment": 5501, + "environmental": 7831, + "environmentally": 32855, + "environments": 19577, + "envision": 49031, + "envoy": 29263, + "envy": 21017, + "eny": 20482, + "enya": 36509, + "enyc": 39520, + "enz": 25805, + "enz": 31873, + "enza": 25239, + "enzie": 14839, + "enzo": 31543, + "enzyme": 40348, + "enzymes": 47465, + "eo": 16054, + "eo": 11712, + "eoin": 48634, + "eon": 31915, + "eos": 17805, + "ep": 1178, + "ep": 1117, + "epa": 15866, + "epage": 26931, + "epaper": 33584, + "epcot": 32524, + "eper": 43071, + "eph": 45752, + "eph": 41240, + "ephe": 25129, + "epi": 7219, + "epi": 34641, + "epic": 12683, + "epic": 4991, + "epiconetsy": 49222, + "epide": 17382, + "epidemi": 44447, + "epidemic": 21522, + "epile": 23150, + "epilepsy": 29547, + "epilo": 31291, + "epilots": 39766, + "epiph": 40561, + "epiphany": 43251, + "epis": 24616, + "episcop": 28037, + "episcopal": 31221, + "episo": 2708, + "episode": 2965, + "episodes": 11837, + "epit": 21967, + "epitome": 35114, + "epl": 25950, + "epo": 25810, + "epp": 39054, + "epp": 39593, + "eps": 4090, + "epsilon": 40019, + "epsom": 40364, + "epstein": 34688, + "eq": 39331, + "eq": 33692, + "equ": 2563, + "equal": 17373, + "equal": 10433, + "equality": 48981, + "equality": 9578, + "equally": 18172, + "equals": 30278, + "equation": 28591, + "equations": 38225, + "eque": 19518, + "equestrian": 24728, + "equi": 8752, + "equili": 43262, + "equine": 33801, + "equinox": 32652, + "equip": 6526, + "equip": 36979, + "equipment": 6893, + "equipo": 45688, + "equipped": 18331, + "equitable": 44717, + "equities": 44015, + "equity": 11293, + "equivalent": 19489, + "er": 517, + "er": 528, + "era": 30548, + "era": 2072, + "erable": 18801, + "erad": 24194, + "eradic": 36346, + "eradicate": 46164, + "eral": 6222, + "eran": 13069, + "eras": 19325, + "eras": 39090, + "erase": 33893, + "erased": 46762, + "erasmus": 38935, + "erc": 5360, + "erc": 32382, + "erd": 25645, + "erdo": 21112, + "erdogan": 24453, + "ere": 17907, + "ere": 642, + "erec": 21526, + "erected": 39365, + "ered": 9097, + "eres": 15751, + "ergon": 38120, + "ergy": 19550, + "eri": 2769, + "eri": 9509, + "eria": 11634, + "erial": 5409, + "eric": 1206, + "eric": 5396, + "erica": 13208, + "erich": 26070, + "erick": 27434, + "erick": 36959, + "erickson": 45286, + "ericsson": 39645, + "eridge": 45408, + "erie": 7005, + "eries": 9099, + "erik": 22805, + "erik": 16532, + "erika": 25531, + "erin": 17532, + "erin": 11333, + "erina": 25176, + "ering": 1785, + "erit": 23335, + "eritrea": 30738, + "erjee": 41665, + "erly": 14380, + "erm": 31649, + "erman": 17990, + "ern": 6992, + "ern": 12140, + "ernal": 20868, + "ernan": 34617, + "ernation": 48796, + "erne": 33930, + "ernest": 23006, + "ernie": 23636, + "ernity": 14653, + "erno": 40812, + "ernst": 30099, + "ero": 3211, + "ero": 3732, + "erock": 38206, + "eron": 32837, + "eroom": 46690, + "eros": 30597, + "erose": 48657, + "erosion": 30174, + "erotic": 30708, + "erotica": 39126, + "erous": 6384, + "eroy": 36461, + "erp": 28268, + "err": 22479, + "err": 25346, + "erra": 48446, + "errands": 45485, + "error": 12097, + "errors": 21195, + "erry": 45236, + "erry": 24124, + "ers": 4840, + "ers": 612, + "ersfc": 37925, + "ership": 2884, + "erson": 25780, + "erson": 6811, + "ert": 40325, + "ert": 3112, + "erta": 32007, + "erton": 26245, + "erts": 12921, + "eru": 36068, + "erun": 41642, + "erup": 17093, + "erupted": 48862, + "eruption": 33705, + "erville": 37557, + "erwin": 43724, + "ery": 12467, + "ery": 1692, + "erz": 38711, + "es": 957, + "es": 542, + "esa": 46834, + "esa": 12489, + "esanders": 23099, + "esc": 3330, + "esc": 28420, + "escal": 15902, + "escap": 11499, + "escape": 32484, + "escape": 7568, + "escaped": 18707, + "escapes": 29916, + "escaping": 21767, + "escar": 39229, + "escence": 37972, + "esch": 46760, + "esch": 41945, + "esco": 32482, + "escobar": 48807, + "escor": 24360, + "escort": 24976, + "escorted": 47667, + "escorts": 48574, + "escu": 36517, + "esday": 19553, + "ese": 18766, + "ese": 2260, + "esg": 41674, + "esh": 17119, + "esh": 13407, + "esha": 28799, + "eshop": 38451, + "eshop": 45570, + "eshopsuk": 39349, + "esi": 30064, + "esis": 12414, + "esk": 19359, + "esl": 26201, + "eso": 29890, + "eso": 28921, + "esof": 17047, + "eson": 46845, + "esp": 3849, + "esp": 13870, + "espa": 37301, + "espan": 41731, + "españa": 41118, + "especially": 4878, + "esper": 29216, + "espino": 46633, + "espionage": 43498, + "espn": 22917, + "espn": 7540, + "espnu": 47747, + "espo": 34381, + "esports": 16035, + "espresso": 17098, + "esq": 47352, + "esqu": 34616, + "esque": 25877, + "ess": 3118, + "ess": 9764, + "essa": 39125, + "essay": 12751, + "essays": 27328, + "esse": 22305, + "essen": 30489, + "essence": 17830, + "essenti": 11163, + "essential": 47264, + "essential": 6895, + "essentially": 30042, + "essentials": 16191, + "essex": 30563, + "essex": 11623, + "est": 2291, + "est": 1509, + "esta": 41449, + "esta": 10135, + "estab": 7010, + "establi": 8412, + "establish": 19709, + "established": 13143, + "establishing": 29420, + "establishment": 20213, + "estas": 39072, + "estate": 47130, + "estate": 6159, + "estates": 26054, + "este": 12968, + "este": 20579, + "esteban": 48381, + "esteem": 31541, + "esteemed": 36293, + "ester": 45808, + "esthe": 18468, + "esther": 24393, + "estim": 8904, + "estimate": 21883, + "estimated": 16665, + "estimates": 21957, + "esto": 31589, + "esto": 23958, + "estonia": 26260, + "estonian": 48895, + "estrada": 48116, + "estre": 31271, + "estu": 26272, + "estuary": 35269, + "esur": 35758, + "esville": 39187, + "esy": 46268, + "et": 1169, + "et": 875, + "eta": 8761, + "etal": 25221, + "etary": 13074, + "etc": 5353, + "etched": 40411, + "etching": 41375, + "ete": 38820, + "ete": 40245, + "eter": 8587, + "eter": 17007, + "eternal": 13732, + "eternally": 48486, + "eternity": 23832, + "eters": 18392, + "etf": 31661, + "eth": 4819, + "eth": 5927, + "ethan": 24245, + "ethan": 15958, + "ethanol": 38166, + "ethe": 21312, + "ethel": 45921, + "ether": 23349, + "ethere": 18705, + "ethereal": 40925, + "ethereum": 19612, + "ethernet": 35026, + "ethi": 10327, + "ethic": 39104, + "ethical": 47041, + "ethical": 17679, + "ethics": 13355, + "ethiop": 10897, + "ethiopia": 13920, + "ethiopian": 24507, + "ethnic": 30522, + "ethnic": 16344, + "ethnicity": 46787, + "ethno": 34225, + "ethos": 48768, + "eti": 11188, + "eti": 30394, + "etienne": 46118, + "eties": 15137, + "etihad": 38489, + "etiquette": 37957, + "etis": 38216, + "etisation": 39733, + "etna": 41940, + "eto": 27829, + "eto": 33837, + "eton": 44339, + "etour": 41462, + "etr": 23012, + "etres": 42838, + "ets": 3442, + "etsy": 13237, + "etsy": 6282, + "etsym": 22902, + "etsymntt": 25416, + "etsyshop": 44643, + "ett": 32729, + "ett": 24998, + "etta": 30466, + "ette": 19981, + "ette": 5212, + "ettes": 35326, + "etto": 44219, + "etty": 40759, + "etu": 36593, + "etv": 49155, + "etv": 20325, + "etwork": 20585, + "ety": 25920, + "ety": 2746, + "etz": 36181, + "etz": 25301, + "eu": 1506, + "eu": 3238, + "eucalyp": 41068, + "eucalyptus": 42351, + "euchar": 38362, + "eugen": 30678, + "eugene": 17760, + "eul": 46749, + "eun": 16431, + "eun": 26219, + "eunhyuk": 47526, + "eup": 44435, + "euph": 21386, + "euphoria": 41051, + "eur": 18343, + "eur": 12018, + "eura": 32605, + "eure": 25311, + "euref": 48017, + "eureka": 31686, + "euro": 2039, + "euro": 8463, + "euroleague": 46821, + "europa": 18290, + "europale": 42473, + "europaleague": 44029, + "europarl": 44922, + "europe": 4198, + "europe": 3848, + "european": 26712, + "european": 4759, + "europeans": 37082, + "euros": 22274, + "eurovision": 17593, + "eurozone": 42555, + "eurusd": 40895, + "eus": 44214, + "euston": 46905, + "euthan": 43280, + "euve": 40652, + "eux": 25019, + "ev": 776, + "ev": 10133, + "eva": 6845, + "evacu": 13187, + "evacuated": 26806, + "evacuation": 27353, + "eval": 25139, + "eval": 9703, + "evalu": 10314, + "evaluate": 27174, + "evaluating": 34541, + "evaluation": 17640, + "evan": 12821, + "evan": 12847, + "evangel": 20518, + "evangeli": 21372, + "evangelical": 36151, + "evangelist": 42275, + "evankirstel": 46581, + "evans": 8836, + "evansville": 44782, + "evapor": 33352, + "evasion": 48795, + "eve": 5732, + "eve": 1866, + "eved": 19820, + "evel": 39315, + "evelyn": 26687, + "evement": 8210, + "even": 6359, + "even": 1427, + "evening": 34487, + "evening": 2285, + "evenings": 19994, + "evenly": 45974, + "event": 10612, + "event": 1655, + "eventful": 45628, + "evento": 38155, + "eventprofs": 24980, + "events": 3667, + "eventu": 14055, + "eventual": 45321, + "eventually": 14397, + "ever": 888, + "ever": 1247, + "everest": 21722, + "everett": 25456, + "everglades": 46294, + "evergreen": 23852, + "everlasting": 32849, + "evers": 31914, + "everton": 13315, + "every": 1091, + "every": 1505, + "everybody": 5901, + "everyday": 25049, + "everyday": 5160, + "everyone": 1584, + "everything": 36376, + "everything": 2410, + "everytime": 16911, + "everywhere": 6364, + "eves": 7323, + "evi": 5348, + "evi": 36989, + "evic": 21336, + "eviction": 37111, + "eviden": 46220, + "evidence": 6439, + "evident": 34529, + "evie": 47195, + "evil": 23218, + "evil": 6006, + "eville": 16143, + "eving": 24729, + "evo": 17962, + "evo": 13169, + "evoc": 43133, + "evol": 5350, + "evolu": 7725, + "evolution": 8902, + "evolutionary": 30629, + "evolve": 23406, + "evolved": 22613, + "evolving": 23675, + "evp": 46154, + "evs": 33576, + "ew": 11942, + "ew": 15428, + "ewan": 40247, + "ewe": 48438, + "ewing": 38873, + "ews": 9878, + "ex": 659, + "ex": 4118, + "exac": 5460, + "exact": 12651, + "exactly": 5840, + "exagger": 29766, + "exal": 49324, + "exam": 4428, + "exam": 8785, + "examination": 20970, + "examine": 25728, + "examined": 44004, + "examiner": 29149, + "examines": 28160, + "examining": 30616, + "example": 6228, + "examples": 14790, + "exams": 14028, + "exas": 47536, + "exc": 1302, + "excav": 20733, + "excavation": 45909, + "exce": 10999, + "exceed": 32521, + "exceeded": 36221, + "exceeding": 47213, + "exceeds": 49353, + "excel": 28351, + "excel": 18754, + "excell": 3298, + "excellence": 8171, + "excellency": 36503, + "excellent": 4239, + "excelsi": 47315, + "excep": 8882, + "except": 8541, + "exception": 25018, + "exceptional": 13425, + "exceptionally": 29306, + "excer": 17737, + "excerpt": 20586, + "excess": 22491, + "excessive": 21332, + "exchange": 6616, + "exchanged": 48919, + "exchanges": 29730, + "exchanging": 47760, + "excit": 10510, + "excite": 47711, + "excited": 1889, + "excitement": 11407, + "exciting": 4300, + "exclu": 3114, + "exclude": 49235, + "excluded": 46216, + "excluding": 44326, + "exclusion": 40219, + "exclusive": 3747, + "exclusively": 13565, + "exclusives": 47149, + "excu": 7324, + "excur": 27533, + "excursion": 34869, + "excuse": 9266, + "excuses": 19388, + "exe": 3554, + "exe": 48027, + "exec": 15052, + "execs": 35728, + "execu": 4360, + "execute": 36405, + "executed": 20432, + "execution": 18085, + "executive": 5944, + "executives": 24357, + "exem": 19753, + "exemp": 28602, + "exempl": 36371, + "exemplary": 39123, + "exempli": 41934, + "exempt": 44278, + "exemption": 47481, + "exer": 40295, + "exerc": 5932, + "exercise": 7016, + "exercises": 19669, + "exercising": 39036, + "exeter": 32137, + "exeter": 18837, + "exfoli": 38823, + "exhau": 11154, + "exhaust": 21812, + "exhausted": 21741, + "exhausting": 40035, + "exhaustion": 49221, + "exhi": 3022, + "exhib": 3783, + "exhibit": 24992, + "exhibit": 8209, + "exhibiting": 23889, + "exhibition": 4219, + "exhibitions": 28311, + "exhibitor": 44192, + "exhibitors": 38542, + "exhibits": 30093, + "exhilar": 40262, + "exhilarating": 49289, + "exi": 5297, + "exico": 38712, + "exile": 28566, + "exist": 10899, + "exist": 9645, + "existed": 23198, + "existence": 13832, + "existent": 43541, + "existential": 38752, + "existing": 12886, + "exists": 14608, + "exit": 9374, + "exited": 37581, + "exiting": 39577, + "exits": 34943, + "exmoor": 48260, + "exo": 15600, + "exo": 5842, + "exodus": 30098, + "exol": 42856, + "exop": 35288, + "exoplan": 37980, + "exor": 24506, + "exorcist": 46309, + "exotic": 15639, + "exp": 9923, + "exp": 19066, + "expan": 7512, + "expand": 10382, + "expand": 13141, + "expanded": 18390, + "expanding": 15755, + "expands": 22223, + "expanse": 46886, + "expansion": 10138, + "expansive": 49261, + "expat": 43900, + "expe": 2560, + "expect": 9802, + "expect": 5716, + "expectation": 34273, + "expectations": 12529, + "expected": 5573, + "expecting": 12525, + "expects": 24536, + "expedition": 16761, + "expeditions": 49327, + "expelled": 48834, + "expen": 7216, + "expend": 29302, + "expenditure": 47044, + "expense": 28473, + "expenses": 21797, + "expensive": 9649, + "exper": 1533, + "experi": 4723, + "experience": 31867, + "experience": 2415, + "experienced": 10417, + "experiences": 8233, + "experiencing": 16643, + "experiential": 44952, + "experim": 6697, + "experiment": 13079, + "experimental": 16539, + "experimenting": 28263, + "experiments": 21077, + "expert": 6284, + "expertise": 16555, + "experts": 6960, + "expi": 26850, + "expir": 35077, + "expire": 49315, + "expired": 30200, + "expires": 34739, + "expl": 3261, + "expla": 3517, + "explain": 48918, + "explain": 7304, + "explained": 14229, + "explaining": 13136, + "explains": 6655, + "explan": 13294, + "explanation": 16577, + "explanations": 34383, + "explic": 21011, + "explicit": 33228, + "explo": 3586, + "explode": 31262, + "exploded": 28947, + "explodes": 38119, + "exploding": 34683, + "exploit": 36953, + "exploited": 48554, + "explor": 11958, + "exploration": 14043, + "explore": 10405, + "explore": 5147, + "explorebc": 38754, + "explorecanada": 36600, + "explored": 25016, + "explorer": 15776, + "explorers": 28491, + "explores": 13996, + "exploring": 7584, + "explosion": 13785, + "explosions": 38646, + "explosive": 18888, + "explosives": 44705, + "expo": 7820, + "expo": 6344, + "expon": 27905, + "export": 14444, + "exporting": 47433, + "exports": 20088, + "expose": 23181, + "exposed": 12180, + "exposes": 33575, + "exposing": 28362, + "exposition": 36943, + "exposure": 11903, + "expre": 6085, + "express": 18553, + "express": 5642, + "expressed": 20777, + "expresses": 31931, + "expressing": 30207, + "expression": 11357, + "expressions": 20314, + "expressive": 42060, + "expressway": 31658, + "exquis": 16575, + "exquisite": 17958, + "ext": 5711, + "ext": 20072, + "exten": 5555, + "extend": 14492, + "extended": 9614, + "extending": 25652, + "extends": 20688, + "extension": 10275, + "extensions": 24525, + "extensive": 16870, + "extensively": 47365, + "extent": 24913, + "exter": 9797, + "exterior": 19352, + "extermin": 41671, + "external": 15028, + "extin": 13553, + "extinct": 24488, + "extinction": 21186, + "extingui": 38567, + "extor": 35620, + "extr": 29082, + "extra": 6416, + "extra": 4231, + "extrac": 18550, + "extract": 18962, + "extraction": 28789, + "extracts": 45576, + "extraordin": 23628, + "extraordinaire": 30909, + "extraordinary": 10982, + "extras": 29817, + "extravag": 22299, + "extravaganza": 29461, + "extre": 3978, + "extreme": 38357, + "extreme": 8331, + "extremely": 6519, + "extremism": 31493, + "extremist": 36383, + "extremists": 41425, + "extru": 43010, + "ey": 1541, + "ey": 1477, + "eyang": 28915, + "eye": 5034, + "eye": 3272, + "eyebrow": 34250, + "eyebrows": 19923, + "eyed": 15512, + "eyeing": 34916, + "eyel": 17075, + "eyelashes": 42074, + "eyeliner": 33354, + "eyeon": 25126, + "eyes": 3095, + "eyeshadow": 35213, + "eyewear": 30165, + "eyewitness": 36258, + "eyou": 31996, + "eyour": 40229, + "eyre": 44115, + "ez": 10082, + "ez": 8387, + "eze": 25993, + "eze": 27229, + "ezekiel": 41428, + "ezra": 27552, + "f": 69, + "f": 325, + "fa": 778, + "fa": 2800, + "faa": 27577, + "fab": 2833, + "fab": 5492, + "faber": 43461, + "faber": 42488, + "fabi": 29425, + "fabian": 34539, + "fabio": 31666, + "fabric": 16217, + "fabric": 10033, + "fabricated": 40851, + "fabrication": 33476, + "fabrics": 23159, + "fabulous": 5189, + "fac": 1053, + "fac": 35438, + "facade": 29217, + "face": 2545, + "face": 1710, + "facebook": 36156, + "facebook": 2943, + "faced": 10941, + "faceli": 32023, + "facelift": 36380, + "faceoff": 42710, + "facep": 45285, + "faces": 4905, + "faceted": 43435, + "facetime": 24076, + "facial": 11909, + "facil": 39973, + "facilit": 13567, + "facilitate": 26733, + "facilitated": 43853, + "facilitating": 34796, + "facilities": 10388, + "facility": 8165, + "facing": 7619, + "fact": 17189, + "fact": 3598, + "factfriday": 27953, + "faction": 14629, + "factor": 21082, + "factor": 8124, + "factories": 36492, + "factors": 12733, + "factory": 42483, + "factory": 6072, + "facts": 5085, + "factual": 45471, + "faculty": 9504, + "facup": 25283, + "fad": 12632, + "fad": 47669, + "fade": 20486, + "faded": 26051, + "fades": 40441, + "fading": 32882, + "fadnavis": 38945, + "faf": 31052, + "faf": 43903, + "fag": 25617, + "fag": 39305, + "fah": 25495, + "fah": 35429, + "fahren": 45527, + "fai": 20519, + "fai": 26384, + "fail": 7105, + "fail": 6801, + "failed": 8314, + "failing": 15757, + "fails": 13388, + "failure": 8732, + "failures": 25442, + "faint": 30807, + "fair": 3031, + "fair": 2849, + "fairbanks": 43962, + "faire": 34745, + "faire": 20798, + "fairfax": 29368, + "fairfield": 29664, + "fairgrounds": 38325, + "fairi": 28884, + "fairies": 33590, + "fairly": 14961, + "fairmont": 41547, + "fairness": 29388, + "fairs": 8655, + "fairtrade": 33361, + "fairview": 43479, + "fairway": 44022, + "fairy": 17021, + "fairy": 10444, + "fairytale": 28944, + "fais": 23542, + "faisal": 35459, + "fait": 20567, + "faith": 10653, + "faith": 5080, + "faithful": 15511, + "faiz": 41775, + "fake": 18794, + "fake": 5777, + "faken": 22853, + "fakenews": 26943, + "fakespeare": 49095, + "fal": 2778, + "fal": 40494, + "fala": 47120, + "falcon": 22498, + "falcon": 13571, + "falcons": 13834, + "falk": 34648, + "falkirk": 44080, + "fall": 6489, + "fall": 2359, + "fallen": 8688, + "falling": 48709, + "falling": 7293, + "fallon": 39596, + "fallon": 21281, + "fallontonight": 44627, + "fallout": 49365, + "fallout": 16009, + "falls": 4778, + "falmouth": 38261, + "false": 38948, + "false": 9078, + "falsely": 42321, + "fam": 1058, + "fam": 5128, + "fame": 6573, + "famed": 23302, + "famer": 24554, + "famil": 3395, + "famili": 8488, + "familia": 25622, + "familiar": 10020, + "families": 4612, + "family": 8137, + "family": 1315, + "familyfun": 46308, + "familytime": 47236, + "familytravel": 38222, + "famine": 35847, + "famous": 44811, + "famous": 4096, + "famously": 44505, + "fan": 1675, + "fan": 2261, + "fanart": 41059, + "fanart": 7855, + "fanartfriday": 45346, + "fanatic": 36643, + "fanatics": 39610, + "fanbase": 36921, + "fanboy": 43369, + "fanc": 29017, + "fancafe": 45080, + "fanci": 35908, + "fanclub": 31530, + "fancy": 47622, + "fancy": 6733, + "fand": 19684, + "fandom": 47634, + "fandom": 11534, + "fanfest": 42916, + "fanfic": 47243, + "fang": 14269, + "fang": 27428, + "fangirl": 28813, + "fangirling": 39463, + "fanning": 37282, + "fanny": 30401, + "fans": 32454, + "fans": 1840, + "fansign": 25288, + "fant": 4467, + "fanta": 2703, + "fantaken": 39412, + "fantasia": 49306, + "fantastic": 31289, + "fantastic": 2935, + "fantasy": 15124, + "fantasy": 5267, + "fantasyfootball": 35713, + "fao": 31155, + "faq": 28533, + "far": 1578, + "far": 2384, + "fara": 48562, + "farage": 28340, + "farah": 31547, + "fare": 8620, + "fare": 6461, + "fares": 27525, + "farewell": 10734, + "fargo": 18870, + "fari": 26197, + "farley": 43761, + "farm": 9066, + "farm": 3985, + "farmer": 19735, + "farmer": 10474, + "farmers": 29752, + "farmers": 6402, + "farmersmarket": 41808, + "farmhouse": 26293, + "farming": 10399, + "farmington": 49305, + "farmland": 45258, + "farms": 11277, + "farn": 27527, + "faroo": 39147, + "farra": 33657, + "farrakhan": 46293, + "farrell": 24234, + "fart": 34664, + "farther": 42233, + "fas": 4830, + "fas": 42995, + "fasci": 17191, + "fascin": 7327, + "fascinated": 32964, + "fascinating": 8640, + "fascism": 28213, + "fascist": 23870, + "fascists": 43598, + "fash": 42682, + "fashi": 2099, + "fashion": 6976, + "fashion": 2444, + "fashionable": 24597, + "fashionblogger": 31726, + "fashioned": 21563, + "fashioni": 26062, + "fashionista": 30415, + "fashions": 37601, + "fashionshow": 45653, + "fashionweek": 28684, + "fass": 42398, + "fast": 8509, + "fast": 1953, + "fasten": 44990, + "faster": 8835, + "fastest": 9808, + "fasting": 24656, + "fat": 4751, + "fat": 5484, + "fatal": 12124, + "fatalities": 44168, + "fatally": 34069, + "fate": 26315, + "fate": 11734, + "father": 11607, + "father": 3224, + "fathers": 12780, + "fathersday": 16731, + "fati": 13430, + "fatigue": 23747, + "fatima": 28202, + "fats": 30151, + "fatt": 44131, + "fatty": 22953, + "fau": 5571, + "fau": 31381, + "faucet": 44273, + "faul": 16230, + "faulkner": 37840, + "fault": 13862, + "faults": 42752, + "faulty": 47103, + "fauna": 30808, + "faust": 44772, + "faux": 19429, + "fav": 1355, + "fav": 5426, + "fave": 7272, + "faves": 18003, + "favor": 1766, + "favor": 12160, + "favorable": 35392, + "favored": 46640, + "favorite": 35262, + "favorite": 1916, + "favorited": 36926, + "favorites": 10564, + "favors": 36085, + "favour": 3111, + "favour": 20469, + "favourite": 3342, + "favourites": 16585, + "favs": 18879, + "faw": 21800, + "fawad": 46425, + "fawn": 48624, + "fax": 32535, + "fax": 9337, + "fay": 8939, + "fay": 40074, + "faye": 30257, + "fayette": 32043, + "fayette": 19782, + "fayetteville": 37771, + "fayre": 34982, + "faz": 26238, + "faze": 44880, + "fb": 22637, + "fb": 3307, + "fball": 29663, + "fbf": 20004, + "fbi": 10293, + "fbloggers": 41389, + "fbs": 48454, + "fc": 4278, + "fc": 1399, + "fca": 24540, + "fcb": 26639, + "fcb": 25045, + "fcbarcelona": 32174, + "fcbayern": 35033, + "fcblive": 44608, + "fcc": 21240, + "fck": 40080, + "fck": 49263, + "fcofficial": 27805, + "fcs": 32095, + "fcu": 47898, + "fd": 16972, + "fd": 11525, + "fda": 17823, + "fdi": 45579, + "fdn": 18563, + "fdny": 41084, + "fdr": 42298, + "fe": 623, + "fe": 873, + "fear": 8744, + "fear": 5402, + "feared": 31154, + "fearless": 17470, + "fears": 13867, + "fearthe": 33449, + "feasi": 34977, + "feast": 37963, + "feast": 9564, + "feat": 1703, + "feat": 5611, + "feather": 24905, + "feather": 17871, + "feathers": 21138, + "featherweight": 44939, + "feature": 30413, + "feature": 4527, + "featured": 4743, + "features": 4643, + "featuring": 3706, + "feb": 4317, + "febru": 4202, + "february": 4248, + "fect": 31293, + "fed": 22518, + "fed": 7035, + "feder": 4737, + "federal": 6369, + "federation": 15530, + "federer": 18246, + "federico": 40539, + "fedex": 32603, + "fedora": 45111, + "feds": 30593, + "fee": 28242, + "fee": 9224, + "feed": 6662, + "feed": 5839, + "feedback": 8683, + "feeder": 24482, + "feeders": 44523, + "feeding": 9879, + "feeds": 21788, + "feel": 2408, + "feel": 2051, + "feelin": 19903, + "feeling": 33087, + "feeling": 3045, + "feelings": 9452, + "feels": 4808, + "feelthe": 22322, + "feelthebern": 27743, + "fees": 11765, + "feet": 4804, + "fei": 23441, + "fei": 34217, + "fein": 46707, + "feinstein": 41313, + "fel": 2081, + "fel": 20304, + "feld": 45913, + "feld": 14219, + "feldman": 41942, + "feli": 7498, + "felic": 25845, + "felici": 23379, + "felicia": 41139, + "felicidades": 41648, + "felicity": 35123, + "feline": 29471, + "felipe": 27681, + "felix": 33455, + "felix": 16514, + "feliz": 26104, + "feliz": 20221, + "fell": 33540, + "fell": 6266, + "fella": 17586, + "fellas": 18787, + "feller": 29226, + "fellow": 12099, + "fellow": 5242, + "fellows": 15766, + "fellowship": 13857, + "felony": 31068, + "felt": 5413, + "fem": 24574, + "fem": 36615, + "fema": 41721, + "female": 22062, + "female": 3970, + "females": 21028, + "femi": 38607, + "femin": 11423, + "femini": 11894, + "feminine": 24911, + "feminism": 18784, + "feminist": 14921, + "feminists": 38809, + "femme": 31331, + "fen": 5509, + "fen": 25024, + "fence": 12679, + "fences": 34312, + "fencing": 23489, + "fender": 17117, + "fener": 41208, + "fenerbah": 46652, + "feng": 33291, + "fennel": 28689, + "fent": 26395, + "fenton": 47265, + "fenway": 29206, + "fer": 1765, + "fer": 2897, + "fera": 37705, + "feral": 29972, + "ferdin": 25541, + "ferdinand": 27591, + "fere": 43144, + "feren": 35652, + "ference": 19984, + "ferg": 44938, + "fergie": 39119, + "fergu": 10988, + "fergus": 42041, + "ferguson": 11904, + "fermentation": 45817, + "fermented": 36886, + "fern": 10747, + "fern": 21685, + "fernandes": 44391, + "fernandez": 23436, + "fernando": 17140, + "ferns": 38277, + "feroci": 45652, + "ferr": 7256, + "ferra": 47911, + "ferrari": 9606, + "ferre": 29626, + "ferred": 10432, + "ferreira": 48686, + "ferrell": 41112, + "ferrer": 38904, + "ferri": 42008, + "ferries": 28489, + "ferris": 27532, + "ferry": 38936, + "ferry": 10278, + "fers": 12378, + "fert": 14925, + "fert": 43662, + "fertil": 41987, + "fertile": 44837, + "fertili": 23912, + "fertility": 23528, + "fertilizer": 36786, + "fery": 47448, + "fes": 32300, + "fest": 17383, + "fest": 2590, + "festa": 42124, + "festi": 1943, + "festiv": 19222, + "festival": 20946, + "festival": 2240, + "festivals": 17834, + "festive": 9533, + "festivities": 21020, + "fet": 21409, + "feta": 31705, + "fetal": 42031, + "fetch": 30271, + "fete": 34629, + "fett": 37979, + "fetus": 26768, + "feu": 24912, + "feu": 32990, + "feud": 27365, + "fever": 40896, + "fever": 9989, + "fevre": 43861, + "few": 1939, + "fewer": 19128, + "fex": 41584, + "fex": 26392, + "fey": 39069, + "fey": 23298, + "fez": 43081, + "ff": 1021, + "ff": 1304, + "ffa": 15355, + "ffame": 42873, + "ffc": 19832, + "ffe": 1138, + "ffe": 8631, + "ffect": 29151, + "ffed": 8448, + "ffee": 26377, + "ffel": 22656, + "ffen": 46537, + "ffer": 27369, + "ffer": 11636, + "ffers": 32163, + "fferty": 44771, + "ffes": 46441, + "ffey": 30138, + "fff": 28106, + "ffi": 19961, + "ffic": 4762, + "ffice": 26044, + "ffici": 3639, + "fficial": 39818, + "fficial": 6463, + "fficiency": 27800, + "fficient": 20424, + "ffin": 12779, + "ffin": 7367, + "ffing": 16592, + "ffins": 17898, + "ffl": 39490, + "ffle": 7749, + "ffler": 39819, + "ffles": 19344, + "ffman": 15823, + "ffo": 42264, + "ffs": 4424, + "ffxiv": 26569, + "ffxv": 46786, + "ffy": 26404, + "ffy": 7795, + "fg": 45977, + "fg": 6823, + "fgm": 32178, + "fgo": 46113, + "fh": 21649, + "fh": 21010, + "fhs": 45094, + "fi": 701, + "fi": 3589, + "fia": 8827, + "fiable": 34373, + "fianc": 27752, + "fiance": 44114, + "fiancé": 34039, + "fiasco": 40944, + "fiat": 16740, + "fiawec": 39485, + "fib": 40594, + "fiba": 34993, + "fiber": 35074, + "fiber": 12612, + "fibers": 44587, + "fibre": 21401, + "fibro": 21294, + "fibrosis": 36307, + "fic": 1788, + "fic": 2059, + "fica": 26952, + "fically": 14854, + "fication": 4523, + "fications": 12512, + "ficial": 48192, + "fics": 42505, + "fiction": 6218, + "fictional": 25570, + "fid": 34197, + "fid": 23966, + "fidd": 25218, + "fiddle": 35968, + "fide": 45375, + "fidel": 21740, + "fidel": 36837, + "fidelity": 30109, + "fidget": 48664, + "fie": 28487, + "fie": 10348, + "fied": 29642, + "fied": 2853, + "fiel": 1361, + "field": 7571, + "field": 1570, + "fielder": 11046, + "fieldhouse": 37969, + "fielding": 30465, + "fields": 6494, + "fieldwork": 33155, + "fiends": 37869, + "fier": 11167, + "fier": 10598, + "fierc": 48609, + "fierce": 13896, + "fiercely": 49039, + "fiers": 16113, + "fiery": 24557, + "fies": 9537, + "fiesta": 14580, + "fif": 5309, + "fifa": 21976, + "fifa": 8516, + "fifaworldcup": 38819, + "fifawwc": 41329, + "fife": 24374, + "fifteen": 29504, + "fifth": 25515, + "fifth": 8772, + "fifthharmony": 31075, + "fifty": 24456, + "fifty": 15978, + "fig": 4814, + "fig": 20719, + "figaro": 48044, + "figh": 23274, + "fight": 5262, + "fight": 2757, + "fighter": 35884, + "fighter": 6438, + "fighters": 7371, + "fightfor": 48909, + "fightfor": 35740, + "fighting": 38625, + "fighting": 4652, + "fighton": 45578, + "fights": 12132, + "figs": 38882, + "figu": 6390, + "figur": 16948, + "figurative": 44042, + "figure": 48820, + "figure": 5274, + "figured": 15630, + "figures": 8739, + "figurine": 33306, + "figuring": 31513, + "fiji": 48270, + "fiji": 18285, + "fik": 46589, + "fil": 1142, + "fil": 14915, + "fila": 30992, + "filament": 49252, + "file": 12545, + "file": 4512, + "filed": 13864, + "files": 7850, + "filet": 43155, + "fili": 9590, + "filing": 16576, + "filip": 14368, + "filipino": 19153, + "fill": 15904, + "fill": 6277, + "filled": 5589, + "filler": 32816, + "fillers": 45005, + "fillet": 39276, + "filling": 9736, + "fillion": 38048, + "fillmore": 43922, + "fills": 21750, + "filly": 27690, + "film": 5117, + "film": 1860, + "filmed": 15801, + "filmfare": 42224, + "filmfest": 24508, + "filmfestival": 28066, + "filming": 6866, + "filmmaker": 17202, + "filmmakers": 24896, + "filmmaking": 18226, + "films": 5370, + "fils": 40271, + "filter": 7541, + "filtered": 29926, + "filtering": 47770, + "filters": 18385, + "filth": 39713, + "filthy": 26899, + "filtr": 21408, + "filtration": 42036, + "fim": 47525, + "fin": 735, + "fin": 10663, + "fina": 34497, + "final": 11968, + "final": 1755, + "finale": 7844, + "finalfantasy": 44543, + "finalfour": 46999, + "finalist": 12620, + "finalists": 13422, + "finalized": 48930, + "finally": 1992, + "finals": 4536, + "finan": 4807, + "finance": 6117, + "finances": 28767, + "financi": 12846, + "financial": 19783, + "financial": 4930, + "financially": 28124, + "financing": 18375, + "finch": 18523, + "find": 18638, + "find": 1416, + "finder": 15045, + "finders": 43884, + "findia": 47064, + "finding": 37455, + "finding": 6002, + "findings": 16529, + "findlay": 48227, + "findom": 36463, + "finds": 6680, + "findyour": 25936, + "findyourpark": 38924, + "fine": 12042, + "fine": 3797, + "fineart": 7484, + "fineart": 16005, + "fineartamerica": 7724, + "fined": 20094, + "finely": 46120, + "finer": 36681, + "fines": 25053, + "finesse": 46047, + "finest": 7707, + "fing": 6485, + "fing": 17955, + "finger": 13480, + "finger": 8895, + "fingerprint": 39579, + "fingers": 9690, + "fini": 2405, + "finish": 42178, + "finish": 3958, + "finished": 3078, + "finisher": 38636, + "finishers": 48661, + "finishes": 13078, + "finishing": 7912, + "finite": 48312, + "finity": 41463, + "finity": 21273, + "fink": 40158, + "finland": 10775, + "finley": 41652, + "finn": 28479, + "finn": 16925, + "finna": 35180, + "finnish": 19616, + "fino": 30083, + "fins": 32810, + "fintech": 48929, + "fintech": 8899, + "fion": 27476, + "fiona": 20099, + "fior": 37086, + "fiore": 44997, + "fioren": 33188, + "fiorentina": 43713, + "fios": 42521, + "fir": 770, + "fir": 16233, + "fire": 2951, + "fire": 1769, + "firearm": 40311, + "firearms": 23960, + "fireball": 40543, + "firec": 42806, + "fired": 8846, + "firefighter": 20498, + "firefighters": 12600, + "firefly": 33997, + "firefox": 35372, + "fireman": 46085, + "firen": 34752, + "firenze": 38445, + "fireplace": 23050, + "fires": 8749, + "fireside": 36185, + "firework": 40750, + "fireworks": 10641, + "firing": 15105, + "firm": 16936, + "firm": 7705, + "firmly": 29156, + "firms": 13655, + "firmware": 42691, + "first": 6853, + "first": 874, + "firstdayof": 44297, + "firsth": 48512, + "firsts": 47884, + "firth": 26078, + "fis": 7846, + "fis": 47683, + "fiscal": 20825, + "fischer": 26532, + "fish": 6431, + "fish": 2759, + "fisher": 11175, + "fisher": 9176, + "fisheries": 24612, + "fisherman": 25055, + "fishermen": 28547, + "fishers": 42065, + "fishery": 49057, + "fishes": 35470, + "fishing": 31703, + "fishing": 4935, + "fishy": 35665, + "fist": 48340, + "fist": 17085, + "fit": 2366, + "fit": 2478, + "fitbit": 33768, + "fitch": 44614, + "fitfam": 20662, + "fitnes": 47285, + "fitness": 20044, + "fitness": 4838, + "fits": 6401, + "fitt": 32994, + "fitted": 14863, + "fitter": 42096, + "fitters": 32364, + "fitting": 11769, + "fittings": 45787, + "fitz": 11120, + "fitz": 25913, + "fitzgerald": 20606, + "fitzpatrick": 37141, + "fiu": 38374, + "five": 19508, + "five": 3127, + "fives": 44066, + "fix": 4596, + "fix": 6028, + "fixed": 9393, + "fixes": 25473, + "fixing": 17423, + "fixture": 17317, + "fixtures": 19904, + "fizz": 31242, + "fj": 43183, + "fj": 46447, + "fjor": 31260, + "fk": 12410, + "fl": 1082, + "fl": 2685, + "fla": 1577, + "fla": 20292, + "flag": 11536, + "flag": 4859, + "flagged": 45012, + "flags": 12221, + "flagship": 19779, + "flagstaff": 40406, + "flair": 24938, + "flake": 21221, + "flakes": 20934, + "flam": 10559, + "flame": 40351, + "flame": 13484, + "flamen": 28826, + "flamenco": 37362, + "flames": 13441, + "flamin": 42693, + "flaming": 34782, + "flamingo": 30323, + "flan": 14572, + "flanagan": 28641, + "flanders": 34837, + "flank": 44553, + "flann": 39510, + "flannel": 37807, + "flap": 35253, + "flappy": 40241, + "flare": 21185, + "flares": 46088, + "flash": 6089, + "flash": 5815, + "flashback": 14616, + "flashback": 11988, + "flashbackfriday": 15014, + "flashbacks": 47056, + "flashes": 31259, + "flashing": 31764, + "flashlight": 37256, + "flask": 36194, + "flat": 8986, + "flat": 6313, + "flats": 17228, + "flatt": 45498, + "flattering": 43267, + "flaun": 41421, + "flav": 7191, + "flavo": 28895, + "flavor": 31835, + "flavor": 11818, + "flavored": 29350, + "flavorful": 49135, + "flavors": 16930, + "flavour": 17026, + "flavoured": 42397, + "flavours": 21083, + "flaw": 14268, + "flaw": 34978, + "flawed": 35136, + "flawless": 15531, + "flaws": 30492, + "flax": 43443, + "fle": 2428, + "fle": 44964, + "flea": 24883, + "fleck": 28143, + "fled": 26731, + "flee": 19427, + "flee": 30167, + "fleece": 25038, + "fleeing": 30543, + "fleek": 43513, + "fleet": 35922, + "fleet": 9147, + "fleetwood": 28883, + "fleming": 25769, + "fler": 48789, + "flesh": 17495, + "flet": 16102, + "fletcher": 19810, + "fleur": 28593, + "flew": 13768, + "flex": 16426, + "flex": 12038, + "flexi": 10032, + "flexibility": 22547, + "flexible": 14502, + "flexing": 48483, + "fli": 2472, + "flick": 13746, + "flick": 23414, + "flickr": 17755, + "flies": 8070, + "flight": 24701, + "flight": 3795, + "flights": 10515, + "flin": 24730, + "flin": 43816, + "flinders": 44647, + "fling": 22768, + "flint": 28306, + "flint": 18324, + "flip": 20385, + "flip": 11035, + "flipk": 30829, + "flipkart": 33154, + "flipped": 28144, + "flipping": 25881, + "flips": 35089, + "flir": 24330, + "flirt": 38352, + "flirting": 35243, + "flix": 40663, + "flo": 1945, + "flo": 20711, + "float": 16123, + "floating": 12619, + "floats": 33272, + "flock": 36297, + "flock": 21822, + "flondon": 47366, + "floo": 4062, + "flood": 23793, + "flood": 7148, + "flooded": 19706, + "flooding": 10204, + "floods": 16369, + "floor": 23657, + "floor": 4125, + "flooring": 19227, + "floors": 15671, + "flop": 22994, + "floppy": 38267, + "flops": 29146, + "flor": 15784, + "flor": 41669, + "flora": 18906, + "floral": 10732, + "florals": 48331, + "floren": 37706, + "florence": 11617, + "flores": 21537, + "flori": 3482, + "florian": 41861, + "florida": 34264, + "florida": 3966, + "florist": 38403, + "floss": 36453, + "flotus": 35181, + "flour": 18592, + "flouri": 23239, + "flourish": 36038, + "flow": 2180, + "flow": 5608, + "flower": 12772, + "flower": 4055, + "flowering": 19953, + "flowers": 4023, + "flowing": 14922, + "flown": 25659, + "flows": 16715, + "floyd": 46369, + "floyd": 13656, + "flu": 3698, + "flu": 13528, + "fluctu": 40181, + "fluence": 38169, + "fluent": 30025, + "fluff": 31174, + "fluffy": 40346, + "fluffy": 17054, + "fluid": 43803, + "fluid": 16717, + "fluids": 41490, + "fluor": 45127, + "fluore": 26974, + "fluorescent": 35036, + "fluori": 45611, + "flur": 31591, + "flush": 25777, + "flushing": 43754, + "flute": 23746, + "flux": 25249, + "flwx": 30907, + "fly": 5666, + "fly": 3228, + "flye": 30873, + "flyeagles": 39927, + "flyeaglesfly": 39931, + "flyer": 11875, + "flyers": 14181, + "flyfishing": 31800, + "flying": 20782, + "flying": 4610, + "flyn": 40676, + "flynn": 15721, + "flyo": 33506, + "flyover": 38083, + "fm": 13715, + "fm": 3689, + "fman": 25152, + "fml": 26730, + "fmr": 32875, + "fn": 22773, + "fn": 21763, + "fnc": 46506, + "fo": 898, + "fo": 6157, + "foal": 40386, + "foam": 30039, + "foam": 14587, + "foamed": 26711, + "fob": 40315, + "focal": 30934, + "focu": 5827, + "focus": 4353, + "focused": 9319, + "focuses": 20093, + "focusing": 15551, + "fod": 31015, + "fod": 43299, + "fodils": 44411, + "foe": 22952, + "foes": 46279, + "fog": 9417, + "foggy": 19770, + "foil": 17302, + "fol": 1106, + "fol": 48616, + "fold": 35201, + "fold": 11021, + "foldable": 48307, + "folded": 25233, + "folder": 25717, + "folding": 15464, + "folds": 24266, + "foley": 22850, + "foli": 7713, + "folia": 48964, + "foliage": 26350, + "folio": 10772, + "folk": 10665, + "folk": 6032, + "folke": 47190, + "folkl": 27273, + "folklore": 22133, + "folklore": 28620, + "folklorethursday": 23270, + "folks": 5422, + "follo": 41417, + "follow": 1964, + "follow": 1979, + "followart": 40957, + "followback": 33863, + "followed": 6499, + "follower": 17039, + "followers": 4856, + "following": 3473, + "followme": 29668, + "followparty": 44757, + "follows": 11287, + "followthe": 30747, + "folly": 41408, + "folsom": 42108, + "fom": 34540, + "fon": 5017, + "fon": 38318, + "fond": 19964, + "fonda": 44609, + "fondue": 48321, + "fone": 40672, + "font": 37610, + "font": 16248, + "fontaine": 37864, + "fontana": 43643, + "fontein": 45062, + "fonts": 32801, + "foo": 1183, + "foo": 23435, + "food": 4586, + "food": 1559, + "foodand": 38317, + "foodbank": 31926, + "foodie": 30762, + "foodie": 9847, + "foodies": 22416, + "foodnetwork": 46793, + "foods": 7057, + "foodsecurity": 49329, + "foodtruck": 47682, + "fool": 23959, + "fool": 12212, + "fooled": 28761, + "fooling": 47964, + "foolish": 33824, + "fools": 15946, + "foot": 6702, + "foot": 4738, + "footage": 11130, + "footb": 33466, + "football": 9376, + "football": 1882, + "footballer": 20646, + "footballers": 30269, + "footed": 38040, + "footh": 25951, + "foothills": 37020, + "footpath": 48858, + "footprint": 23206, + "footprints": 39640, + "footsteps": 27289, + "footwear": 22772, + "footy": 39866, + "footy": 18922, + "for": 645, + "for": 556, + "forage": 46871, + "foraging": 39056, + "forall": 17824, + "forbe": 49098, + "forbes": 13925, + "forbi": 24754, + "forbidden": 25164, + "force": 12068, + "force": 2869, + "forced": 8201, + "forces": 5381, + "forchange": 35848, + "forcing": 21573, + "ford": 3751, + "ford": 1623, + "fordfc": 28581, + "fordham": 48792, + "fords": 29351, + "fordshire": 14645, + "fore": 1484, + "fore": 1332, + "forec": 34155, + "forecast": 7361, + "forecasting": 38133, + "forecasts": 27696, + "foreclo": 44916, + "forefront": 37679, + "foreground": 35186, + "forehead": 25394, + "foreig": 26497, + "foreign": 42255, + "foreign": 6046, + "foreigners": 38549, + "foreman": 36174, + "foremost": 42128, + "foren": 16526, + "forensic": 23158, + "forensics": 38763, + "forest": 18760, + "forest": 4167, + "forestation": 33939, + "forestry": 26281, + "forests": 14095, + "forever": 14748, + "forever": 3225, + "forevery": 40605, + "forex": 40200, + "forex": 17395, + "forfe": 44871, + "forge": 19232, + "forged": 28105, + "forget": 46153, + "forget": 2678, + "forgets": 35613, + "forgetting": 25452, + "forgi": 22080, + "forgive": 15332, + "forgiven": 44894, + "forgiveness": 23585, + "forgood": 39169, + "forgot": 6483, + "forgotten": 7994, + "fork": 24501, + "fork": 13700, + "forkids": 48571, + "forklift": 43202, + "forks": 28769, + "forlife": 17624, + "form": 1157, + "form": 1907, + "forma": 38829, + "formal": 12978, + "formally": 24867, + "format": 16252, + "format": 11874, + "formation": 2510, + "formations": 37715, + "formative": 48882, + "formats": 32085, + "forme": 42085, + "formed": 6528, + "former": 2276, + "formerly": 20866, + "formid": 38599, + "formidable": 39834, + "forming": 15443, + "formity": 42290, + "forms": 5161, + "formu": 8689, + "formul": 23923, + "formula": 24485, + "formula": 10776, + "formulae": 34586, + "formulated": 45066, + "forre": 38876, + "forrest": 25205, + "forrester": 45338, + "forsa": 48958, + "forsale": 13303, + "forster": 42923, + "forsy": 29629, + "forsyth": 40952, + "fort": 12300, + "fort": 2921, + "forte": 44350, + "forte": 27367, + "forth": 17068, + "forth": 11932, + "forthcoming": 19989, + "forthe": 12521, + "forti": 26984, + "fortified": 46486, + "fortn": 14428, + "fortnight": 39235, + "fortnite": 38734, + "fortnite": 17890, + "fortress": 19988, + "fortun": 6950, + "fortunate": 19898, + "fortunately": 34358, + "fortune": 40931, + "fortune": 11451, + "fortunes": 41989, + "forty": 24399, + "forum": 37851, + "forum": 4538, + "forums": 31518, + "forwar": 34364, + "forward": 47031, + "forward": 2342, + "forwards": 38974, + "foryou": 35150, + "forz": 46056, + "forza": 33293, + "forza": 28089, + "fos": 36925, + "fos": 22081, + "foss": 14240, + "foss": 37911, + "fossil": 20419, + "fossil": 15202, + "fossilfriday": 26079, + "fossils": 30652, + "foster": 26778, + "foster": 8139, + "fostering": 35996, + "fosters": 37644, + "foto": 15908, + "foto": 12823, + "fotogra": 23687, + "fotografia": 40256, + "fotos": 26124, + "fou": 14516, + "fought": 10844, + "foul": 19784, + "foun": 3154, + "found": 3454, + "found": 1546, + "foundation": 4058, + "foundations": 25219, + "founded": 12240, + "founder": 5145, + "founders": 14602, + "founding": 15317, + "foundry": 31426, + "fountain": 44863, + "fountain": 13405, + "fountains": 37411, + "four": 5113, + "four": 2721, + "foursquare": 34484, + "fourteen": 46255, + "fourth": 7516, + "fourthofjuly": 47805, + "fow": 17084, + "fowl": 31685, + "fowler": 20980, + "fox": 5007, + "fox": 3240, + "foxandfriends": 45841, + "foxes": 24145, + "foxnews": 18830, + "foxsports": 39267, + "foxtv": 49396, + "foxx": 32993, + "foxy": 27945, + "foy": 30284, + "foyer": 38011, + "foyle": 47902, + "fp": 28058, + "fp": 8941, + "fpl": 27970, + "fpp": 36464, + "fps": 25300, + "fpv": 43175, + "fr": 936, + "fr": 5512, + "fra": 3368, + "fra": 15644, + "frac": 15607, + "fracking": 21894, + "fractal": 46471, + "fraction": 26788, + "fractu": 25847, + "fracture": 28995, + "fractured": 37421, + "fractures": 46213, + "frag": 13093, + "fragile": 23579, + "fragment": 39209, + "fragments": 41424, + "fragr": 15403, + "fragrance": 17874, + "fragrances": 44567, + "fragrant": 37030, + "fram": 27987, + "frame": 11029, + "frame": 6481, + "framed": 13135, + "frames": 15479, + "framework": 13195, + "frameworks": 43136, + "framing": 24539, + "frampton": 41733, + "fran": 2118, + "fran": 18878, + "franc": 3872, + "franc": 42340, + "franca": 48952, + "france": 12045, + "france": 3552, + "frances": 20803, + "francesca": 32327, + "francesco": 25816, + "franch": 11756, + "franchi": 46438, + "franchise": 13664, + "franci": 46458, + "francis": 22187, + "francis": 7660, + "francisco": 6887, + "franco": 17934, + "franco": 17052, + "francois": 29317, + "frank": 5390, + "frank": 5229, + "franken": 20487, + "franken": 48252, + "frankenstein": 26410, + "frankfur": 17442, + "frankfurt": 18598, + "franki": 39227, + "frankie": 38373, + "frankie": 16215, + "franklin": 40935, + "franklin": 9999, + "frankly": 38015, + "franks": 42855, + "frans": 47892, + "franz": 25449, + "franç": 38381, + "fraser": 39082, + "fraser": 16754, + "frat": 15225, + "frat": 39292, + "fraternity": 24433, + "frau": 23063, + "fraud": 40647, + "fraud": 9961, + "fraudul": 42655, + "fraudulent": 47408, + "fray": 41154, + "frazier": 32841, + "frc": 41507, + "fre": 821, + "fre": 43165, + "freak": 20352, + "freak": 13701, + "freaked": 43511, + "freakin": 23900, + "freaking": 11992, + "freaks": 27009, + "freaky": 31583, + "freck": 33328, + "freckles": 48036, + "fred": 9486, + "fred": 6678, + "freddie": 41890, + "freddie": 17014, + "freddy": 24394, + "freder": 10745, + "frederic": 41165, + "frederick": 37103, + "frederick": 18570, + "fredo": 48241, + "free": 2065, + "free": 1139, + "freebie": 35865, + "freebies": 28630, + "freec": 46569, + "freed": 12585, + "freed": 23392, + "freedom": 17992, + "freedom": 4511, + "freedoms": 32500, + "freef": 48678, + "freel": 14174, + "freelance": 21942, + "freely": 24436, + "freeman": 16450, + "freep": 32499, + "freepalestine": 39242, + "freer": 44676, + "frees": 27455, + "freestyle": 15594, + "freeway": 24927, + "freeze": 14187, + "freezer": 25390, + "freezing": 12499, + "frei": 30183, + "freight": 17023, + "fremantle": 48012, + "fremont": 34578, + "fren": 2919, + "french": 13118, + "french": 3461, + "frenzy": 30084, + "frequ": 9211, + "frequencies": 45319, + "frequency": 18825, + "frequent": 19836, + "frequently": 22434, + "fresco": 31609, + "fresh": 4065, + "fresh": 2975, + "fresher": 49284, + "freshers": 35810, + "freshest": 46809, + "freshly": 16081, + "freshman": 9381, + "freshmen": 21292, + "freshness": 45872, + "freshwater": 24803, + "fresno": 40879, + "fresno": 20995, + "fret": 40510, + "freud": 40787, + "frey": 22136, + "frey": 9082, + "fri": 815, + "fri": 6882, + "friars": 30513, + "fric": 18981, + "frick": 46304, + "friction": 38563, + "frid": 46388, + "frida": 36001, + "friday": 6350, + "friday": 1461, + "fridayfeeling": 11952, + "fridaymotivation": 38544, + "fridaynight": 44858, + "fridayreads": 37736, + "fridays": 15589, + "fridaythe": 47642, + "fridge": 13491, + "fridges": 40734, + "frie": 36999, + "fried": 13743, + "fried": 7310, + "friedman": 29402, + "friedrich": 34171, + "friend": 3017, + "friend": 1625, + "friendly": 44612, + "friendly": 4681, + "friends": 38875, + "friends": 1574, + "friendship": 42674, + "friendship": 7679, + "friendships": 28840, + "fries": 11369, + "frifotos": 40493, + "friger": 20785, + "friggin": 48300, + "frigh": 34831, + "fright": 24277, + "fright": 40207, + "frightened": 47136, + "frightening": 39290, + "fringe": 10640, + "fris": 37252, + "frisbee": 45768, + "frisco": 35945, + "frit": 34614, + "fritz": 29860, + "friyay": 38887, + "frm": 12951, + "fro": 626, + "fro": 26603, + "frock": 45306, + "frog": 26494, + "frog": 11438, + "frogs": 20781, + "from": 8330, + "from": 633, + "frome": 48691, + "fromhome": 41477, + "fromthe": 18756, + "fron": 1847, + "fron": 18036, + "front": 10996, + "front": 2184, + "frontal": 35794, + "frontier": 18253, + "frontiers": 38396, + "frontline": 29589, + "frontman": 36775, + "fronts": 26846, + "froome": 48560, + "frosh": 47069, + "frost": 39420, + "frost": 11619, + "frosted": 35988, + "frosting": 33872, + "frosty": 22760, + "froze": 47788, + "frozen": 42464, + "frozen": 8507, + "frs": 26216, + "fru": 3248, + "fruit": 16771, + "fruit": 5190, + "fruitful": 31494, + "fruits": 13282, + "fruity": 22320, + "frustr": 16046, + "frustrated": 25111, + "frustrating": 31342, + "frustration": 30535, + "fry": 33914, + "fry": 13686, + "fryer": 49217, + "frying": 38516, + "fs": 23699, + "fs": 3854, + "fsa": 33373, + "fsu": 44185, + "fsu": 19317, + "ft": 3391, + "ft": 981, + "fta": 41975, + "ftc": 33752, + "fted": 5612, + "fter": 25063, + "fthe": 22886, + "ftheday": 9823, + "fting": 6174, + "fton": 26605, + "ftp": 42649, + "fts": 3767, + "ftse": 46717, + "ftw": 19298, + "fty": 17494, + "fu": 665, + "fu": 9098, + "fuch": 42617, + "fudge": 24270, + "fue": 43723, + "fuego": 41500, + "fuel": 21113, + "fuel": 5945, + "fueled": 28792, + "fueling": 38793, + "fuelled": 48357, + "fuels": 19365, + "fuentes": 44393, + "fuer": 29645, + "fug": 29227, + "fugitive": 39257, + "fuji": 15573, + "fuji": 21634, + "fujifilm": 24765, + "fuk": 31051, + "fuku": 20728, + "fukushima": 33929, + "ful": 1814, + "ful": 857, + "fulbright": 41834, + "fulfill": 43675, + "fulfill": 27467, + "fulfilled": 29919, + "fulfilling": 30621, + "fulfillment": 45573, + "fulham": 25574, + "full": 9407, + "full": 1476, + "fuller": 20225, + "fullerton": 42822, + "fullest": 35603, + "fully": 39142, + "fully": 2401, + "fulness": 10526, + "fuls": 41606, + "fulton": 26725, + "fum": 38393, + "fumble": 49373, + "fun": 1229, + "fun": 1499, + "func": 8679, + "function": 8093, + "functional": 12885, + "functionality": 33316, + "functioning": 25479, + "functions": 18001, + "fund": 19089, + "fund": 4877, + "fundam": 11670, + "fundament": 18852, + "fundamental": 17627, + "fundamentally": 45378, + "fundamentals": 27887, + "funday": 15439, + "funded": 10588, + "funding": 5588, + "fundra": 6201, + "fundraiser": 10049, + "fundraising": 10755, + "funds": 7066, + "funer": 40693, + "funeral": 10606, + "funfact": 31596, + "funfactfriday": 40710, + "fungal": 38838, + "fungi": 27837, + "fungus": 30677, + "funk": 37353, + "funk": 13372, + "funko": 49402, + "funko": 23697, + "funky": 16492, + "funnel": 27862, + "funnier": 42232, + "funniest": 15557, + "funny": 19124, + "funny": 3789, + "funrun": 34185, + "fur": 2395, + "fur": 9686, + "furi": 40816, + "furious": 17522, + "furman": 49238, + "furn": 21348, + "furnace": 31913, + "furnished": 37388, + "furnitu": 45696, + "furniture": 7993, + "furry": 33414, + "furry": 15351, + "fursuit": 25306, + "fursuit": 43083, + "fursuitfriday": 27917, + "further": 5583, + "fury": 14404, + "fus": 18419, + "fuse": 23386, + "fused": 38994, + "fusion": 44661, + "fusion": 9364, + "fuss": 26331, + "fut": 21460, + "fut": 34049, + "futbol": 33014, + "futsal": 20558, + "futu": 33454, + "futur": 38840, + "future": 7959, + "future": 1904, + "futureof": 22599, + "futureofwork": 33202, + "futures": 13488, + "futuri": 19068, + "futurism": 48435, + "futurist": 48086, + "futuristic": 30987, + "fuzz": 47128, + "fuzz": 40443, + "fuzzy": 25876, + "fv": 29795, + "fw": 23934, + "fw": 5277, + "fwd": 27052, + "fx": 17807, + "fx": 9025, + "fy": 8440, + "fy": 2702, + "fyi": 16014, + "fying": 5294, + "fz": 46400, + "fé": 34072, + "g": 70, + "g": 326, + "ga": 1275, + "ga": 1531, + "gaa": 10715, + "gaal": 40867, + "gaard": 24645, + "gab": 3927, + "gab": 37382, + "gabbana": 36272, + "gabby": 48115, + "gabby": 24567, + "gabe": 18916, + "gabi": 41931, + "gable": 33387, + "gables": 40928, + "gabri": 8311, + "gabriel": 31684, + "gabriel": 13244, + "gabrielle": 33572, + "gaby": 46420, + "gac": 32520, + "gad": 7786, + "gad": 44651, + "gadget": 25525, + "gadgets": 22840, + "gado": 29489, + "gae": 22003, + "gael": 35663, + "gaelic": 31173, + "gaf": 21354, + "gaf": 32670, + "gag": 14121, + "gag": 18844, + "gaga": 9782, + "gage": 21081, + "gah": 27750, + "gai": 24214, + "gai": 25153, + "gaia": 41269, + "gail": 41160, + "gail": 27676, + "gain": 21536, + "gain": 6202, + "gaine": 35747, + "gained": 14489, + "gaines": 49225, + "gainesville": 40427, + "gaining": 15260, + "gains": 42751, + "gains": 12107, + "gal": 2001, + "gal": 4488, + "gala": 7211, + "galac": 18864, + "galactic": 25514, + "galap": 41115, + "galapagos": 44057, + "galat": 39853, + "galatasar": 42413, + "galatasaray": 47787, + "galax": 5647, + "galaxies": 32435, + "galaxy": 32130, + "galaxy": 6545, + "gale": 37658, + "gale": 21380, + "galerie": 44539, + "gales": 48633, + "gali": 17546, + "gali": 30552, + "galicia": 47927, + "galileo": 39671, + "gall": 3011, + "gall": 33374, + "galla": 16847, + "gallagher": 19168, + "galleria": 40656, + "galleries": 22304, + "gallery": 36648, + "gallery": 3830, + "galley": 48917, + "galli": 22568, + "gallipoli": 47249, + "gallo": 37350, + "gallo": 33265, + "gallon": 24615, + "gallons": 29335, + "galloway": 27796, + "galore": 22286, + "gals": 20125, + "galvani": 46046, + "galve": 34328, + "galveston": 36003, + "galway": 38045, + "galway": 17112, + "gam": 1162, + "gam": 34195, + "gama": 35873, + "gambia": 32988, + "gamble": 26121, + "gambling": 20287, + "game": 2882, + "game": 1063, + "gameart": 31490, + "gameboy": 40951, + "gamecube": 44079, + "gameday": 9241, + "gamedev": 7544, + "gameinsight": 42626, + "gameof": 10987, + "gameofthrones": 11822, + "gameon": 47691, + "gameplay": 16794, + "gamer": 12595, + "gamer": 11598, + "gamergate": 25961, + "gamers": 16166, + "gamersunite": 26423, + "games": 18551, + "games": 1955, + "gamescom": 37003, + "gamestop": 39436, + "gametime": 45899, + "gami": 42025, + "gamification": 48908, + "gaming": 28803, + "gaming": 4017, + "gamma": 22180, + "gamo": 39325, + "gan": 1822, + "gan": 1670, + "gand": 8399, + "ganda": 27261, + "gander": 44508, + "gandhi": 12322, + "ganesh": 30362, + "ganesha": 45185, + "gang": 8066, + "gang": 5674, + "ganga": 36275, + "gangnam": 46777, + "gangs": 29844, + "gangsta": 37365, + "gangster": 26514, + "gani": 48324, + "gann": 45665, + "gannon": 45837, + "gano": 25304, + "gao": 26556, + "gaon": 19279, + "gap": 29906, + "gap": 7609, + "gaps": 25296, + "gar": 1099, + "gar": 5824, + "gara": 28710, + "garage": 8474, + "garbage": 13760, + "garci": 44658, + "garcia": 10529, + "gard": 7751, + "gard": 21003, + "garda": 31906, + "garde": 22649, + "garden": 4674, + "garden": 2756, + "gardenchat": 46292, + "gardener": 28554, + "gardeners": 38205, + "gardening": 10483, + "gardens": 6152, + "gardiner": 43121, + "gardner": 18710, + "gare": 5633, + "gare": 48402, + "gareth": 37140, + "gareth": 18175, + "garfield": 26728, + "garh": 16762, + "gari": 40898, + "gari": 43080, + "garis": 37839, + "garland": 23418, + "garlic": 9685, + "garment": 31418, + "garments": 43341, + "garmin": 39885, + "garner": 20340, + "garnet": 37669, + "garo": 30388, + "garrett": 15881, + "garri": 21764, + "garrison": 30108, + "garros": 40425, + "garry": 24398, + "gars": 12055, + "gart": 18380, + "gart": 18751, + "garten": 14684, + "garter": 48420, + "garth": 45398, + "garth": 24469, + "gartner": 43334, + "gartner": 29678, + "garty": 46383, + "garu": 31140, + "garvey": 39511, + "garwal": 38623, + "gary": 10535, + "gary": 4516, + "garza": 49393, + "gas": 5047, + "gas": 2474, + "gases": 36971, + "gasoline": 27691, + "gasp": 43762, + "gaston": 40669, + "gastri": 49197, + "gastro": 23740, + "gastron": 30699, + "gastronomy": 46987, + "gat": 5314, + "gat": 18941, + "gata": 44575, + "gate": 8071, + "gate": 3302, + "gated": 23997, + "gates": 9472, + "gateshead": 40051, + "gateway": 45221, + "gateway": 14943, + "gather": 36345, + "gather": 12602, + "gathered": 14646, + "gathering": 9197, + "gatherings": 48096, + "gathers": 39250, + "gating": 27561, + "gation": 11095, + "gations": 33906, + "gato": 44492, + "gator": 20216, + "gator": 16390, + "gatorade": 36354, + "gators": 17173, + "gatory": 24796, + "gatsby": 32586, + "gatwick": 37122, + "gau": 5919, + "gau": 43068, + "gauge": 18728, + "gaunt": 31862, + "gauntlet": 37163, + "gautam": 45853, + "gautam": 31356, + "gauteng": 40333, + "gav": 8966, + "gave": 3485, + "gavin": 32974, + "gavin": 16389, + "gaw": 15405, + "gawd": 43239, + "gawx": 43420, + "gay": 7460, + "gay": 5627, + "gaya": 39477, + "gaye": 41401, + "gayle": 29998, + "gayo": 36768, + "gays": 28001, + "gaz": 4837, + "gaz": 36475, + "gaza": 38391, + "gaza": 10112, + "gazaunderattack": 42458, + "gaze": 23212, + "gazette": 20443, + "gazing": 28373, + "gb": 8727, + "gb": 4619, + "gba": 18528, + "gbbo": 34474, + "gbc": 42993, + "gbp": 27391, + "gbr": 31984, + "gby": 40509, + "gc": 8577, + "gc": 6043, + "gcc": 26804, + "gcse": 28763, + "gcu": 34137, + "gd": 13264, + "gd": 14604, + "gdc": 32793, + "gden": 44928, + "gdp": 17100, + "gdpr": 22963, + "ge": 619, + "ge": 710, + "gea": 26790, + "gear": 15532, + "gear": 4802, + "gearbox": 42454, + "geared": 33903, + "gearing": 19027, + "gears": 21147, + "geaux": 36313, + "gecko": 38616, + "ged": 17252, + "ged": 3480, + "geddon": 31720, + "gedly": 13991, + "gee": 9806, + "gee": 9071, + "geek": 17920, + "geek": 7135, + "geeks": 20110, + "geeky": 47332, + "geel": 25906, + "geelong": 34555, + "gees": 38088, + "geese": 26413, + "geez": 42394, + "geh": 30320, + "geist": 38290, + "gel": 7343, + "gel": 5697, + "gelato": 29577, + "gels": 42552, + "gely": 14637, + "gem": 14261, + "gem": 7613, + "gement": 19495, + "gemini": 23086, + "gemma": 23952, + "gems": 14355, + "gemstone": 27747, + "gemstones": 43972, + "gen": 1024, + "gen": 3278, + "gence": 16088, + "gency": 5245, + "gend": 33247, + "gender": 22976, + "gender": 5906, + "gendere": 35824, + "genderequality": 43338, + "gene": 5822, + "gene": 7962, + "genealo": 24142, + "genealogy": 29381, + "gener": 1832, + "general": 20576, + "general": 3658, + "generally": 19256, + "generals": 30296, + "generate": 16896, + "generated": 19450, + "generates": 33938, + "generating": 23882, + "generation": 41211, + "generation": 4883, + "generational": 34506, + "generations": 12247, + "generative": 29472, + "generator": 19399, + "generators": 41917, + "generic": 26978, + "generosity": 23015, + "generous": 12570, + "generously": 35113, + "genes": 19683, + "genesis": 13518, + "genetic": 47746, + "genetic": 13578, + "genetically": 36745, + "genetics": 18276, + "geneva": 14799, + "genevie": 41633, + "genevieve": 46584, + "geni": 22334, + "genic": 15750, + "genie": 24221, + "genital": 32960, + "genius": 8235, + "geniuses": 41406, + "geno": 41544, + "geno": 46776, + "genoa": 43993, + "genoci": 14687, + "genocide": 15903, + "genome": 23991, + "genomic": 44371, + "genomics": 26227, + "genre": 14249, + "genres": 30340, + "gens": 17449, + "gent": 3685, + "gent": 7139, + "gente": 34325, + "gentle": 7262, + "gentle": 13577, + "gentleman": 13293, + "gentlemen": 11692, + "gently": 17187, + "gento": 28320, + "gentri": 41148, + "gentry": 47225, + "gents": 18862, + "genu": 9182, + "genuine": 12184, + "genuinely": 20006, + "genus": 38161, + "geny": 35323, + "geo": 5038, + "geo": 11604, + "geocaching": 47908, + "geof": 20629, + "geoff": 33697, + "geoff": 20386, + "geoffrey": 29520, + "geograph": 45920, + "geographic": 22635, + "geographical": 39380, + "geography": 17101, + "geological": 38380, + "geology": 21578, + "geom": 46135, + "geome": 12958, + "geometric": 22419, + "geometry": 21731, + "geon": 20844, + "geon": 7295, + "geons": 15914, + "geopol": 39758, + "geor": 2549, + "georg": 43126, + "george": 8377, + "george": 3296, + "georges": 25042, + "georgetown": 22970, + "georgie": 42115, + "georgina": 43892, + "geospatial": 46238, + "geothermal": 38413, + "geous": 3068, + "ger": 1291, + "ger": 1502, + "gera": 48867, + "gerald": 29901, + "gerald": 13269, + "gerard": 35979, + "gerard": 20826, + "gerber": 45058, + "gered": 40179, + "geri": 41664, + "geri": 46214, + "gering": 24077, + "germain": 38786, + "german": 14972, + "german": 4710, + "germans": 28400, + "germany": 4464, + "germin": 44721, + "germs": 47731, + "geronimo": 45171, + "gerrard": 26538, + "gerry": 29825, + "gerry": 23026, + "gers": 3314, + "gertrude": 46950, + "gervais": 36527, + "gery": 32845, + "ges": 3316, + "gest": 11843, + "gest": 2033, + "gesture": 21780, + "gestures": 43524, + "get": 5670, + "get": 779, + "geta": 13155, + "getaway": 16131, + "gether": 27224, + "getic": 20661, + "getin": 25822, + "getit": 44891, + "getit": 48315, + "getoutside": 35644, + "gets": 39448, + "gets": 2127, + "gett": 6647, + "gett": 27965, + "gettable": 15620, + "gette": 29800, + "gettin": 13428, + "getting": 30885, + "getting": 1500, + "getty": 31185, + "getty": 13965, + "gettys": 35189, + "gettysburg": 37062, + "getyour": 42159, + "gey": 29289, + "gf": 28953, + "gf": 10846, + "gfriend": 35245, + "gfs": 37553, + "gg": 1129, + "gg": 3286, + "gga": 26003, + "ggan": 25626, + "gge": 21521, + "gge": 31659, + "gged": 6095, + "gger": 12367, + "gger": 3493, + "ggers": 7480, + "ggg": 20143, + "gggg": 33513, + "ggi": 21662, + "ggin": 17160, + "gging": 4966, + "ggins": 12444, + "ggle": 34981, + "ggle": 11430, + "ggled": 46328, + "ggles": 14703, + "ggling": 16523, + "ggly": 39407, + "ggs": 4797, + "ggy": 24935, + "ggy": 6476, + "gh": 583, + "gh": 790, + "gha": 10010, + "gha": 25183, + "gham": 21456, + "ghan": 18945, + "ghan": 6624, + "ghana": 30330, + "ghana": 9731, + "ghanaian": 34223, + "ghani": 36699, + "ghar": 37334, + "ghar": 36973, + "ghat": 43989, + "ghaz": 37493, + "ghc": 42139, + "ghe": 10754, + "ghe": 28561, + "ghead": 40783, + "ghee": 34794, + "gher": 21542, + "gher": 14796, + "ghet": 18447, + "ghetti": 17485, + "ghetto": 22403, + "ghi": 22436, + "ghi": 22279, + "ghibli": 40555, + "ghj": 38439, + "ghlin": 24131, + "gho": 4307, + "ghorn": 38094, + "ghosh": 43279, + "ghoshal": 49134, + "ghost": 11417, + "ghost": 7108, + "ghostbusters": 25462, + "ghostly": 44901, + "ghosts": 16737, + "ghou": 35843, + "ghoul": 45302, + "ghouse": 38238, + "ghs": 14157, + "ght": 1413, + "ght": 630, + "ghted": 4963, + "ghter": 2427, + "ghters": 12994, + "ghtful": 8334, + "ghting": 3019, + "ghtly": 6993, + "ghtning": 39740, + "ghton": 16353, + "ghts": 1259, + "ghty": 20968, + "ghty": 5866, + "ghu": 25808, + "ghue": 45675, + "ghyun": 25010, + "ghz": 24325, + "gi": 707, + "gi": 4478, + "gia": 8864, + "giac": 35444, + "giam": 39623, + "gian": 17274, + "gian": 12866, + "gianni": 46752, + "giant": 23668, + "giant": 4687, + "giants": 7076, + "giar": 34241, + "gib": 9816, + "gibb": 18964, + "gibbons": 31974, + "gibbs": 26488, + "gibility": 33297, + "gible": 13159, + "gibr": 20206, + "gibraltar": 23988, + "gibson": 37420, + "gibson": 12178, + "gic": 27900, + "gic": 2570, + "gical": 32973, + "gically": 26320, + "gid": 36774, + "gid": 21413, + "giddy": 40894, + "gideon": 43867, + "gidi": 30603, + "gie": 11459, + "gie": 3991, + "gier": 28974, + "gies": 5505, + "gif": 11363, + "gif": 11677, + "gifford": 47850, + "gifs": 37643, + "gift": 20569, + "gift": 2733, + "gifted": 15110, + "giftide": 20152, + "giftideas": 23487, + "gifting": 39546, + "gifts": 5836, + "gig": 26981, + "gig": 7471, + "gigab": 34530, + "gigan": 24104, + "gigantic": 31507, + "giggle": 36426, + "giggles": 42731, + "giggs": 44692, + "gigi": 44106, + "gigi": 26171, + "gigs": 20316, + "gil": 3997, + "gil": 10088, + "gila": 46952, + "gilbert": 14154, + "gilded": 44341, + "giles": 24802, + "gill": 14280, + "gill": 12003, + "gille": 29610, + "gilles": 39590, + "gillespie": 36242, + "gillette": 38603, + "gilli": 13695, + "gillian": 28753, + "gills": 48851, + "gilmore": 27603, + "gilt": 44378, + "gim": 31284, + "gimm": 40692, + "gimme": 21525, + "gin": 3374, + "gin": 4941, + "gina": 15604, + "gine": 27482, + "ging": 10829, + "ging": 3905, + "ginger": 16287, + "ginger": 9718, + "gingerbread": 23692, + "gini": 35768, + "gino": 36521, + "gins": 18328, + "gio": 16329, + "gio": 8050, + "gion": 41226, + "gior": 14920, + "giorgio": 33271, + "giorno": 33310, + "gios": 41927, + "gious": 14419, + "giov": 21404, + "giovanni": 26574, + "gipp": 41351, + "gir": 1077, + "gir": 25481, + "gira": 16949, + "giraffe": 22826, + "giri": 31709, + "girl": 3914, + "girl": 1611, + "girlfriend": 8217, + "girlfriends": 30736, + "girlpower": 37433, + "girls": 15480, + "girls": 1917, + "girly": 29605, + "giro": 39664, + "giro": 26454, + "girona": 47842, + "giroud": 41177, + "gis": 16266, + "gis": 12773, + "gist": 21241, + "git": 16060, + "git": 20918, + "gita": 40838, + "github": 31196, + "giu": 17931, + "giuli": 29762, + "giuliani": 47739, + "giuse": 29385, + "giuseppe": 33563, + "give": 4120, + "give": 1781, + "giveaway": 5310, + "giveaways": 18974, + "giveback": 41385, + "given": 33323, + "given": 4302, + "givenchy": 38245, + "giver": 43339, + "gives": 3926, + "giveup": 35485, + "giving": 14673, + "giving": 2339, + "givingback": 49300, + "givingtuesday": 23556, + "giz": 29237, + "gk": 38953, + "gk": 18719, + "gl": 1849, + "gl": 14751, + "gla": 1523, + "gla": 36904, + "glaci": 14924, + "glacial": 40782, + "glacier": 19282, + "glaciers": 42528, + "glad": 20841, + "glad": 4761, + "glades": 37432, + "gladi": 21742, + "gladiator": 38477, + "gladiators": 41087, + "gladly": 41598, + "gladys": 43168, + "glam": 8738, + "glam": 16905, + "glamorous": 22896, + "glamour": 42876, + "glamour": 17499, + "glamping": 46167, + "glan": 40482, + "glan": 45844, + "glance": 26557, + "gland": 41441, + "glar": 48535, + "glar": 41702, + "glare": 46035, + "glas": 29935, + "glas": 43654, + "glasgo": 6757, + "glasgow": 29990, + "glasgow": 7363, + "glass": 16305, + "glass": 3313, + "glasses": 6116, + "glaston": 26848, + "glastonbury": 28233, + "glau": 39171, + "glaze": 28112, + "glazed": 24122, + "gle": 7166, + "gle": 2865, + "glee": 32379, + "glee": 21614, + "glen": 6158, + "glen": 11049, + "glend": 38332, + "glendale": 33043, + "glenn": 32004, + "glenn": 12861, + "gler": 34649, + "gley": 21998, + "gli": 5896, + "gli": 28791, + "glia": 22217, + "glide": 37321, + "glider": 41636, + "glimp": 12888, + "glimpse": 13817, + "glio": 29785, + "glit": 21079, + "glitch": 29563, + "glitter": 16528, + "glitz": 44542, + "glo": 1721, + "glo": 30474, + "glob": 13363, + "global": 6707, + "global": 2779, + "globalgoals": 33211, + "globalhealth": 46751, + "globalization": 47680, + "globally": 17775, + "globalwarming": 46017, + "globe": 19436, + "globe": 9368, + "globes": 38085, + "glock": 38818, + "glomer": 43689, + "gloom": 48594, + "gloomy": 32199, + "glori": 7270, + "gloria": 19244, + "glorious": 9171, + "glory": 36107, + "glory": 7285, + "glos": 40633, + "gloss": 38258, + "gloss": 22014, + "glossy": 29802, + "glou": 15989, + "gloucester": 28133, + "gloucester": 23835, + "gloucestershire": 33789, + "glove": 16078, + "glover": 21594, + "gloves": 12363, + "glow": 30472, + "glow": 10111, + "glowing": 18437, + "glows": 48107, + "glu": 5952, + "glu": 32281, + "glucose": 34642, + "glue": 22103, + "glued": 38135, + "gluten": 15482, + "gluten": 15524, + "glutenfree": 16138, + "gly": 13027, + "glycer": 48914, + "gm": 18743, + "gm": 5918, + "gma": 18155, + "gmail": 11119, + "gman": 41043, + "gman": 36936, + "gmb": 35934, + "gmb": 31799, + "gmbh": 46877, + "gmc": 27257, + "gmo": 23486, + "gms": 36987, + "gmt": 13803, + "gn": 2455, + "gn": 9831, + "gna": 23009, + "gnation": 45912, + "gne": 25407, + "gni": 5104, + "gnment": 25110, + "gno": 23376, + "gno": 43686, + "gnocchi": 48299, + "gnome": 33643, + "gnon": 20561, + "go": 650, + "go": 861, + "goa": 14399, + "goal": 9003, + "goal": 3321, + "goalie": 20723, + "goalkeeper": 16601, + "goals": 3295, + "goalscorer": 43547, + "goaltender": 44151, + "goat": 34082, + "goat": 9530, + "goats": 18393, + "gob": 29559, + "gobeavs": 48285, + "goblin": 26223, + "goblue": 25232, + "gobucks": 29175, + "gocougs": 34202, + "god": 4190, + "god": 1731, + "godawgs": 40436, + "godbless": 46616, + "godbless": 44007, + "godd": 16589, + "goddamn": 28495, + "goddard": 37827, + "goddess": 10808, + "godfather": 26222, + "godfrey": 40148, + "godis": 38521, + "godly": 42438, + "gods": 33620, + "gods": 10328, + "goducks": 35889, + "godzilla": 23369, + "goe": 22084, + "goers": 27784, + "goes": 43581, + "goes": 2635, + "gof": 17537, + "goff": 34399, + "goftheday": 39360, + "gofund": 34445, + "gofundme": 34686, + "gog": 42949, + "goggles": 31027, + "gogh": 19697, + "gogo": 22688, + "gogreen": 36279, + "gohawks": 34884, + "goi": 24917, + "goin": 13939, + "going": 25787, + "going": 1245, + "goku": 29550, + "gol": 1537, + "gol": 18257, + "gola": 41090, + "gold": 4999, + "gold": 2209, + "goldberg": 25161, + "goldcoast": 34634, + "golden": 10763, + "golden": 3878, + "goldeng": 20650, + "goldenglobes": 26842, + "goldfish": 40293, + "goldie": 42805, + "goldman": 27164, + "golds": 30526, + "golds": 40283, + "goldsmith": 40214, + "gole": 41297, + "golf": 9096, + "golf": 3096, + "golfclub": 45742, + "golfer": 24579, + "golfers": 28441, + "golfing": 31379, + "goli": 29265, + "goliath": 41602, + "gom": 7051, + "goma": 46198, + "gomes": 39128, + "gomez": 16433, + "gon": 1854, + "gon": 3379, + "gona": 34835, + "gone": 35135, + "gone": 3601, + "gong": 28486, + "gonna": 2562, + "gonz": 10587, + "gonzaga": 36241, + "gonzale": 17512, + "gonzales": 31265, + "gonzalez": 18198, + "goo": 1381, + "goo": 17882, + "good": 2185, + "good": 886, + "goodbye": 6968, + "goodday": 46284, + "goode": 42076, + "goodfood": 46844, + "goodfriday": 40360, + "goodie": 29213, + "goodies": 13308, + "goodluck": 19718, + "goodman": 24146, + "goodmorning": 14421, + "goodness": 10531, + "goodnight": 8540, + "goodreads": 31629, + "goods": 9340, + "goodtimes": 22570, + "goodvibes": 43146, + "goodwill": 24902, + "goodwin": 28080, + "goodwood": 30008, + "goody": 35937, + "goodyear": 42858, + "goofy": 26879, + "goog": 18581, + "google": 12195, + "google": 3460, + "googled": 40345, + "googleplay": 37309, + "goon": 15267, + "goons": 30440, + "goooo": 35876, + "goooo": 48957, + "goose": 21445, + "goose": 13822, + "goosebumps": 32254, + "gop": 18942, + "gop": 6250, + "gopack": 46995, + "gopackgo": 47719, + "gopal": 47268, + "gopdebate": 39806, + "gopher": 47750, + "gopher": 48905, + "gophers": 31957, + "gopro": 17511, + "gor": 1747, + "gor": 29827, + "gordo": 47707, + "gordon": 20485, + "gordon": 8244, + "gore": 30311, + "gore": 17872, + "gorg": 46815, + "gorge": 35548, + "gorge": 20038, + "gorgeous": 3241, + "gori": 12461, + "goria": 43359, + "gorilla": 37910, + "gorilla": 21994, + "gorman": 35741, + "goro": 44977, + "gory": 7160, + "gos": 20517, + "gos": 5693, + "gosh": 15395, + "gosling": 35320, + "gosp": 9617, + "gospel": 11313, + "goss": 39734, + "goss": 36924, + "gossi": 15684, + "gossip": 18963, + "got": 10125, + "got": 1005, + "gota": 36693, + "gotcha": 43275, + "gote": 49345, + "goth": 48465, + "goth": 20437, + "gotham": 46123, + "gotham": 18299, + "gothic": 15426, + "goti": 9497, + "goto": 39715, + "gots": 35215, + "gott": 5089, + "gott": 36466, + "gotta": 4633, + "gotten": 5889, + "gotti": 41881, + "gotv": 36089, + "gou": 10520, + "gou": 36555, + "gouache": 43314, + "goul": 33187, + "gould": 31087, + "gour": 13580, + "gourmet": 19111, + "gov": 4022, + "gov": 4564, + "gove": 36997, + "govegan": 38886, + "gover": 10471, + "gover": 16759, + "govern": 2351, + "govern": 32404, + "governance": 13386, + "governing": 30946, + "government": 3149, + "governmental": 42609, + "governments": 19582, + "governor": 17459, + "governor": 6630, + "governors": 26881, + "govin": 42451, + "govt": 5345, + "govuk": 28830, + "gow": 21885, + "gow": 33788, + "gowan": 31307, + "gower": 43448, + "gown": 13719, + "gowns": 38029, + "goyal": 35105, + "gp": 19329, + "gp": 5051, + "gpa": 24098, + "gps": 13639, + "gpu": 38561, + "gq": 40286, + "gq": 31324, + "gr": 709, + "gr": 6062, + "gra": 782, + "gra": 15276, + "grab": 4646, + "grabbed": 22856, + "grabbing": 26440, + "grabs": 17076, + "grac": 11323, + "grace": 13225, + "grace": 5142, + "graced": 31894, + "graceful": 25242, + "graces": 38629, + "graci": 11174, + "gracias": 16463, + "gracie": 23235, + "gracing": 37263, + "gracious": 29044, + "grad": 19869, + "grad": 7291, + "gradable": 41529, + "grade": 45435, + "grade": 3394, + "graded": 13823, + "grader": 23930, + "graders": 10930, + "grades": 10838, + "gradient": 36885, + "grading": 19016, + "grads": 17811, + "gradu": 3230, + "gradual": 45210, + "gradually": 32192, + "graduate": 6675, + "graduated": 15128, + "graduates": 12236, + "graduating": 14819, + "graduation": 8060, + "grady": 33980, + "graeme": 30192, + "graf": 46478, + "graf": 39765, + "graff": 10656, + "graffiti": 11676, + "graft": 32698, + "grafton": 47347, + "graham": 19805, + "graham": 7711, + "grail": 37184, + "grain": 44003, + "grain": 12109, + "grains": 25791, + "gral": 25631, + "gram": 2949, + "gram": 2338, + "grammar": 16077, + "grammy": 15388, + "grammys": 18121, + "grams": 6294, + "gran": 3892, + "gran": 14493, + "granada": 31172, + "grand": 3058, + "grand": 2991, + "grandad": 29148, + "grandchildren": 36856, + "granddaughter": 29460, + "grande": 37514, + "grande": 10757, + "grandes": 36382, + "grandfather": 15346, + "grandma": 10525, + "grandmother": 17469, + "grandpa": 14582, + "grandparents": 21311, + "grandprix": 39358, + "grandson": 20766, + "grandstand": 43172, + "grange": 45027, + "grange": 23850, + "granger": 42968, + "granite": 18813, + "grann": 45585, + "granny": 22710, + "granola": 34271, + "grant": 18682, + "grant": 5442, + "granted": 14156, + "granth": 41283, + "grants": 15123, + "grape": 19131, + "grape": 15959, + "grapefruit": 28347, + "grapes": 18580, + "grapevine": 47619, + "graph": 1349, + "graph": 4407, + "graphene": 38387, + "grapher": 14987, + "graphers": 32088, + "graphic": 15653, + "graphic": 4245, + "graphical": 20878, + "graphicdesign": 21907, + "graphics": 9492, + "graphies": 40164, + "graphite": 29447, + "graphs": 24670, + "graphy": 4897, + "grapp": 30843, + "gras": 31517, + "gras": 17584, + "grasp": 34975, + "grass": 11584, + "grass": 5922, + "grasses": 46807, + "grasshopper": 48894, + "grassi": 42294, + "grasso": 34808, + "grassroots": 21991, + "grassy": 44140, + "grat": 9221, + "grate": 32463, + "grateful": 45659, + "grateful": 5730, + "grati": 36402, + "gratis": 33638, + "gratitude": 12614, + "grav": 20663, + "grave": 16606, + "grave": 9981, + "gravel": 27054, + "graves": 17665, + "graveyard": 31176, + "gravit": 26150, + "gravitational": 45268, + "gravity": 47426, + "gravity": 15160, + "gravy": 21225, + "gray": 12703, + "gray": 7048, + "grays": 46848, + "grayson": 45831, + "grayson": 25471, + "grazi": 42427, + "grazie": 38698, + "grazing": 29889, + "grc": 44069, + "gre": 689, + "gre": 17878, + "grease": 24132, + "greasy": 44376, + "great": 3265, + "great": 830, + "greate": 31930, + "greater": 32725, + "greater": 7033, + "greatest": 39080, + "greatest": 4153, + "greatly": 13978, + "greatness": 14189, + "greats": 21855, + "greaves": 42350, + "greco": 39103, + "gree": 9987, + "gree": 30774, + "greece": 6965, + "greed": 26147, + "greedy": 33301, + "greek": 23844, + "greek": 6842, + "greeks": 35866, + "green": 2762, + "green": 1901, + "greenberg": 46662, + "greene": 16383, + "greener": 31169, + "greenery": 42493, + "greenfield": 39924, + "greeng": 42077, + "greenhouse": 20819, + "greening": 48673, + "greenland": 27345, + "greenpeace": 44755, + "greens": 10235, + "greensboro": 33436, + "greenville": 25156, + "greenway": 35205, + "greenwich": 18658, + "greenwood": 25782, + "greer": 34345, + "greet": 11042, + "greet": 11997, + "greeted": 24546, + "greeting": 17754, + "greetings": 11569, + "greets": 25464, + "greg": 6894, + "greg": 7943, + "gregation": 20131, + "gregg": 39422, + "gregg": 22929, + "gregor": 33856, + "gregor": 16177, + "gregory": 16253, + "gren": 13941, + "gren": 20119, + "grenade": 33679, + "grenfell": 42107, + "gres": 39670, + "gress": 2752, + "gret": 30041, + "greta": 33443, + "gretchen": 45516, + "grette": 38774, + "grew": 10451, + "grey": 9190, + "grey": 5046, + "greyhound": 27363, + "greyhounds": 45718, + "greys": 44311, + "greysanatomy": 36833, + "gri": 2169, + "gri": 18484, + "grid": 29067, + "grid": 9882, + "gridi": 41063, + "gridiron": 47786, + "grids": 46500, + "grief": 21058, + "grier": 22016, + "griev": 36400, + "grieving": 42383, + "griez": 47962, + "griezmann": 48396, + "griff": 17855, + "griff": 35551, + "griffi": 28676, + "griffin": 46612, + "griffin": 13161, + "griffith": 24375, + "griffiths": 34182, + "gril": 49091, + "grill": 44083, + "grill": 9519, + "grille": 34748, + "grilled": 10691, + "grilling": 28324, + "grills": 39464, + "grim": 20383, + "grim": 23635, + "grime": 37101, + "grimes": 25057, + "grimm": 27865, + "grims": 34861, + "grimsby": 41513, + "grin": 11033, + "grin": 28697, + "grinch": 40527, + "grind": 25730, + "grind": 11810, + "grinder": 31733, + "grinding": 21541, + "gring": 40135, + "grip": 15521, + "gripping": 34567, + "grips": 27819, + "gris": 29150, + "grit": 22037, + "grit": 22087, + "grits": 44307, + "gritty": 33704, + "grizz": 14877, + "grizz": 44088, + "grizzlies": 25594, + "grizzly": 29676, + "grl": 48005, + "gro": 1464, + "gro": 12691, + "grocer": 11633, + "groceries": 32409, + "grocery": 13826, + "grom": 45284, + "gron": 22345, + "groningen": 45639, + "groo": 9015, + "groom": 39883, + "groom": 22813, + "grooming": 25575, + "groot": 37708, + "groove": 39484, + "groove": 17680, + "grooves": 43954, + "groovy": 30143, + "gros": 26834, + "gros": 32639, + "gross": 31080, + "gross": 11541, + "grosven": 46911, + "grote": 47207, + "grotto": 45260, + "grou": 1582, + "groun": 45110, + "ground": 9558, + "ground": 2461, + "groundbreaking": 21006, + "grounded": 27799, + "grounds": 8454, + "groundwater": 39457, + "group": 19045, + "group": 1771, + "groupe": 47654, + "groups": 6776, + "grouse": 36327, + "grove": 31756, + "grove": 7463, + "grover": 31345, + "groves": 27306, + "grow": 3179, + "grow": 4559, + "grower": 44925, + "growers": 25689, + "growing": 28429, + "growing": 4425, + "growingup": 43433, + "growler": 47096, + "grown": 41762, + "grown": 7120, + "grows": 13352, + "growth": 17925, + "growth": 4026, + "growthhacking": 25963, + "grp": 27321, + "grt": 28557, + "gru": 5957, + "grub": 34019, + "grue": 42047, + "gruesome": 47111, + "grum": 45454, + "grump": 49015, + "grumpy": 23610, + "grun": 16203, + "grunge": 33745, + "gry": 16140, + "gry": 5364, + "gs": 25818, + "gs": 1345, + "gsa": 40433, + "gsc": 47751, + "gshore": 43392, + "gsm": 32181, + "gsp": 49173, + "gst": 22239, + "gt": 16151, + "gt": 4725, + "gta": 14826, + "gta": 15338, + "gtaonline": 27292, + "gtav": 27283, + "gti": 39954, + "gto": 39071, + "gtr": 33407, + "gts": 37338, + "gtx": 35230, + "gu": 700, + "gu": 12916, + "gua": 23751, + "guacam": 37477, + "guacamole": 40115, + "guad": 22966, + "guadal": 46097, + "guadalu": 36994, + "guadalupe": 38360, + "guam": 37325, + "guan": 44191, + "guan": 42406, + "guang": 27019, + "guangzhou": 37857, + "guar": 4119, + "guaran": 9242, + "guarantee": 17421, + "guaranteed": 14731, + "guarantees": 40154, + "guard": 30776, + "guard": 4901, + "guarded": 40602, + "guardi": 12008, + "guardia": 43628, + "guardian": 23713, + "guardian": 9498, + "guardians": 21479, + "guarding": 24966, + "guardiola": 32100, + "guards": 12810, + "guatem": 19423, + "guatemala": 21670, + "guay": 48591, + "guay": 24247, + "gubernat": 41400, + "gubernatorial": 41618, + "gucci": 16779, + "gud": 48061, + "gud": 22378, + "gue": 2030, + "gue": 2917, + "gued": 38893, + "guel": 23146, + "guelph": 27660, + "guer": 10391, + "guern": 29277, + "guernsey": 33982, + "guerra": 38215, + "guerrero": 31967, + "guerrilla": 36715, + "gues": 39971, + "gues": 12601, + "guess": 35506, + "guess": 3135, + "guessed": 28005, + "guesses": 30623, + "guessing": 21891, + "guest": 27349, + "guest": 3781, + "guests": 6212, + "guet": 36797, + "guetta": 45904, + "guez": 12313, + "gug": 31358, + "guggen": 35086, + "guggenheim": 37135, + "gui": 2587, + "gui": 25746, + "guid": 11437, + "guidance": 12508, + "guide": 21845, + "guide": 3555, + "guided": 13194, + "guidelines": 16591, + "guides": 14375, + "guiding": 22759, + "guido": 41818, + "guil": 5008, + "guild": 19755, + "guild": 16597, + "guildford": 34450, + "guildhall": 47224, + "guillau": 41123, + "guillaume": 45394, + "guiller": 33660, + "guillermo": 39524, + "guilt": 26354, + "guilty": 9761, + "guin": 13284, + "guin": 47863, + "guine": 13759, + "guinea": 18537, + "guinness": 16648, + "guire": 18209, + "guise": 42024, + "guit": 3759, + "guitar": 21746, + "guitar": 5084, + "guitarist": 13035, + "guitars": 15023, + "guj": 34935, + "gujar": 12698, + "gujarat": 14714, + "guk": 20280, + "gul": 5530, + "gul": 21350, + "gula": 27426, + "gular": 34969, + "gulf": 22101, + "gulf": 11279, + "gull": 48764, + "gull": 28778, + "gulls": 37501, + "gully": 46112, + "gum": 22041, + "gum": 11235, + "gumb": 40147, + "gumbo": 47126, + "gummy": 34276, + "gums": 46609, + "gun": 2748, + "gun": 3496, + "guna": 43333, + "gundam": 26087, + "gundy": 21162, + "gunman": 32743, + "gunmen": 44738, + "gunn": 27473, + "gunna": 24002, + "gunnar": 45301, + "gunner": 35285, + "gunners": 37788, + "guns": 7591, + "gunsense": 44781, + "gunshot": 49250, + "gunsn": 49028, + "gup": 38632, + "gup": 47335, + "gupta": 15905, + "gur": 3218, + "gur": 30224, + "gura": 46836, + "gurgaon": 33240, + "guri": 43888, + "gurl": 25445, + "gurmee": 35482, + "gurmeetramrahim": 36549, + "guru": 18629, + "guru": 10800, + "gurudev": 48647, + "gus": 8018, + "gust": 24629, + "gusta": 23024, + "gusta": 44196, + "gustav": 32062, + "gustav": 37921, + "gustave": 43170, + "gustavo": 45943, + "gusto": 37937, + "gusts": 20896, + "gusty": 27589, + "gut": 24780, + "gut": 13486, + "guter": 44963, + "guterres": 48738, + "guth": 31696, + "guthrie": 33164, + "gutier": 32773, + "gutierrez": 33739, + "guts": 25983, + "gutted": 26524, + "gutter": 40537, + "guwa": 43063, + "guwahati": 45045, + "guy": 10008, + "guy": 2149, + "guyana": 45215, + "guyen": 28031, + "guys": 43588, + "guys": 1791, + "guyz": 48170, + "guzman": 37960, + "gv": 15462, + "gv": 17336, + "gw": 7172, + "gw": 15717, + "gwen": 32165, + "gwen": 24182, + "gwin": 43005, + "gwy": 32226, + "gwyne": 36923, + "gx": 40227, + "gy": 2168, + "gy": 1164, + "gya": 43214, + "gyan": 43814, + "gye": 21728, + "gyllen": 49348, + "gym": 9902, + "gym": 5222, + "gymna": 13517, + "gymnasium": 42847, + "gymnast": 42658, + "gymnastics": 20116, + "gyn": 39603, + "gyne": 45836, + "gyp": 40053, + "gypsy": 22354, + "gypt": 41921, + "gz": 45937, + "gz": 35841, + "gö": 40778, + "gü": 31907, + "h": 71, + "h": 327, + "ha": 560, + "ha": 1429, + "haa": 26814, + "haal": 35869, + "haan": 36284, + "haar": 45247, + "haar": 35859, + "haas": 27443, + "haasan": 26601, + "hab": 20573, + "hab": 20002, + "haban": 46225, + "haber": 44737, + "habit": 8491, + "habit": 17215, + "habitat": 11747, + "habitats": 35344, + "habits": 14540, + "habs": 27489, + "hac": 20343, + "hace": 43623, + "haci": 40674, + "hack": 6610, + "hack": 11182, + "hackathon": 25182, + "hacked": 19575, + "hacker": 22376, + "hackers": 21498, + "hacking": 12939, + "hackney": 48811, + "hackney": 24928, + "hacks": 19965, + "had": 10660, + "had": 1100, + "hadi": 39058, + "hadid": 26415, + "hadith": 46907, + "hadley": 44995, + "hadn": 21480, + "hadoop": 43868, + "hae": 30723, + "hae": 27193, + "hafi": 39914, + "hag": 26855, + "hag": 43207, + "hagan": 47489, + "hagen": 14664, + "hager": 48773, + "hagg": 26324, + "hague": 28988, + "hah": 18108, + "hah": 13680, + "haha": 1913, + "haha": 3060, + "hahah": 27253, + "hahah": 15441, + "hahaha": 4722, + "hahahah": 37513, + "hahahah": 20096, + "hahahaha": 8058, + "hahahaha": 9501, + "hahahahah": 33334, + "hahahahaha": 16347, + "hahahahahaha": 26487, + "hahahahahahaha": 43653, + "hahahahahahahaha": 36126, + "hahahha": 49205, + "hahn": 35596, + "hai": 8734, + "hai": 5234, + "haider": 42200, + "haiku": 19542, + "hail": 15272, + "hail": 8634, + "hailed": 44604, + "hailey": 27703, + "hailing": 47288, + "hails": 32571, + "hailstate": 35063, + "hain": 23861, + "hair": 4658, + "hair": 2225, + "haircare": 43682, + "haircut": 14711, + "hairdresser": 47468, + "haired": 27202, + "hairs": 27951, + "hairstyle": 22324, + "hairstyles": 40627, + "hairy": 26513, + "haiti": 17368, + "haitian": 37577, + "haj": 27885, + "haj": 43191, + "haji": 41889, + "hajj": 35576, + "hak": 25142, + "hak": 40671, + "haka": 44011, + "hake": 41663, + "hal": 1296, + "hal": 8708, + "hala": 25918, + "halal": 34216, + "halam": 29061, + "halamadrid": 31132, + "halder": 32201, + "hale": 37038, + "hale": 14701, + "halen": 39204, + "halep": 49017, + "haley": 37330, + "haley": 16839, + "half": 7453, + "half": 2349, + "halftime": 13742, + "halfway": 16736, + "hali": 9860, + "hali": 43030, + "halibut": 49030, + "halifax": 13411, + "hall": 6850, + "hall": 2140, + "halla": 29569, + "halle": 27763, + "halle": 32239, + "hallelujah": 36993, + "halli": 32665, + "hallmark": 31040, + "hallmark": 32053, + "hallmarkchannel": 36840, + "hallo": 3463, + "halloffame": 48578, + "halloween": 28537, + "halloween": 3739, + "halls": 18052, + "hallucin": 35385, + "hallway": 26845, + "halo": 33331, + "halo": 11918, + "halsey": 34256, + "halt": 25640, + "halter": 47194, + "halton": 45445, + "ham": 1522, + "ham": 1714, + "hama": 17944, + "hamas": 14818, + "hamburg": 18409, + "hamburger": 33928, + "hamid": 32377, + "hamil": 6725, + "hamill": 45784, + "hamill": 48729, + "hamillhimself": 47324, + "hamilton": 22448, + "hamilton": 7684, + "hamlet": 27722, + "hamlin": 49326, + "hamm": 46110, + "hammer": 15331, + "hammer": 9401, + "hammered": 37251, + "hammers": 35649, + "hammersmith": 42127, + "hammock": 33682, + "hammond": 21761, + "hamont": 18518, + "hamp": 6665, + "hamper": 27692, + "hampshire": 16006, + "hampstead": 37340, + "hampton": 36582, + "hampton": 12285, + "hamptons": 42415, + "hamr": 47979, + "hamradio": 36712, + "hams": 25619, + "hamster": 33313, + "hamstring": 39990, + "hamza": 45762, + "han": 1545, + "han": 3565, + "hana": 16801, + "hand": 1722, + "hand": 2463, + "handbag": 22654, + "handbags": 35667, + "handball": 27988, + "handbook": 25147, + "handcrafted": 22185, + "handed": 10881, + "handedly": 48656, + "handel": 40072, + "handful": 23725, + "handheld": 26812, + "handic": 17812, + "handicap": 27063, + "handicapp": 42349, + "handing": 19196, + "handle": 43681, + "handle": 7245, + "handled": 26824, + "handler": 29097, + "handles": 22124, + "handling": 14071, + "handmade": 18054, + "handmade": 6737, + "handmadehour": 25724, + "handover": 46922, + "hands": 3500, + "handshake": 38418, + "handsome": 7438, + "handwriting": 29986, + "handwritten": 35192, + "handy": 13479, + "hane": 28411, + "hang": 3351, + "hang": 5592, + "hangar": 33439, + "hanged": 40807, + "hanger": 28905, + "hangin": 22670, + "hanging": 4850, + "hangout": 17572, + "hangover": 20755, + "hangs": 21785, + "hani": 39944, + "hani": 18374, + "hank": 35993, + "hank": 17655, + "hanks": 29943, + "hanley": 47284, + "hann": 5584, + "hanna": 10075, + "hannah": 18622, + "hannah": 9142, + "hannel": 43477, + "hanni": 19493, + "hannibal": 25149, + "hannity": 24569, + "hannover": 39976, + "hanoi": 36134, + "hanover": 33246, + "hans": 35172, + "hans": 16628, + "hansen": 19729, + "hanson": 24602, + "hant": 40641, + "hanuk": 32774, + "hanukkah": 34247, + "hanuman": 46975, + "hao": 27184, + "hap": 44981, + "hap": 47988, + "happ": 784, + "happen": 21486, + "happen": 4506, + "happened": 4402, + "happening": 4284, + "happeningnow": 43107, + "happenings": 41998, + "happens": 4988, + "happier": 14118, + "happiest": 13811, + "happily": 17316, + "happiness": 5096, + "happy": 2952, + "happy": 900, + "happybirthday": 9651, + "happybirthday": 12207, + "happydays": 25106, + "happye": 33922, + "happyeaster": 38745, + "happyfathersday": 43534, + "happyfriday": 33340, + "happyhalloween": 28750, + "happyholidays": 32186, + "happyhour": 32036, + "happymonday": 47364, + "happymothersday": 42425, + "happynewyear": 18655, + "happythanksgiving": 40593, + "happyvalentinesday": 42403, + "haps": 9114, + "haq": 32445, + "har": 915, + "har": 5888, + "hara": 10367, + "haram": 35732, + "haram": 22950, + "haran": 27921, + "harare": 43562, + "haras": 26644, + "harass": 16481, + "harassed": 43067, + "harassment": 16641, + "harat": 28984, + "harb": 5856, + "harbaugh": 45220, + "harbor": 40686, + "harbor": 10202, + "harbour": 35430, + "harbour": 10011, + "harcourt": 48093, + "hard": 3312, + "hard": 1626, + "hardcover": 31123, + "harden": 27350, + "harder": 12274, + "hardest": 15258, + "hardin": 43802, + "harding": 24382, + "hardly": 17363, + "hardro": 28126, + "hardrock": 48365, + "hardrock": 40739, + "hards": 44048, + "hardship": 45085, + "hardt": 17922, + "hardware": 11957, + "hardwell": 45572, + "hardwick": 46864, + "hardwood": 28167, + "hardwork": 42554, + "hardwork": 27404, + "hardworking": 28095, + "hardworkpaysoff": 49193, + "hardy": 48179, + "hardy": 14113, + "hare": 27903, + "hare": 18464, + "harga": 39738, + "hari": 25472, + "hari": 8981, + "harlan": 49133, + "harle": 29096, + "harlem": 17771, + "harley": 24702, + "harley": 13632, + "harleydavidson": 39183, + "harlow": 34113, + "harm": 16656, + "harm": 14452, + "harman": 42434, + "harmed": 39637, + "harmful": 21725, + "harmless": 44369, + "harmon": 10828, + "harmon": 28729, + "harmony": 10785, + "harms": 46703, + "harne": 43323, + "harness": 23205, + "harold": 16917, + "harp": 27339, + "harper": 31288, + "harper": 12634, + "harri": 6639, + "harrier": 37372, + "harriet": 27154, + "harrington": 34340, + "harris": 25356, + "harris": 6925, + "harrisburg": 40590, + "harrison": 34389, + "harrison": 10540, + "harro": 18939, + "harrogate": 30842, + "harrow": 38807, + "harry": 11094, + "harry": 3600, + "harrypotter": 23375, + "harsh": 30596, + "harsh": 16944, + "hart": 9335, + "hart": 7752, + "hartford": 23434, + "harth": 35619, + "hartle": 47482, + "hartley": 31268, + "hartman": 43294, + "haru": 35099, + "harvard": 28118, + "harvard": 12848, + "harve": 6405, + "harvest": 44495, + "harvest": 8971, + "harvested": 35899, + "harvesting": 26674, + "harvey": 33289, + "harvey": 9586, + "harvick": 46983, + "haryana": 27661, + "has": 13855, + "has": 791, + "hasan": 30049, + "hasbro": 37405, + "hash": 6338, + "hash": 19199, + "hashi": 41831, + "hashmi": 35852, + "hashtag": 34015, + "hashtag": 9238, + "hashtags": 23514, + "haskell": 48550, + "hasn": 9143, + "hass": 9298, + "hassan": 15829, + "hassee": 37117, + "hassel": 32204, + "hassle": 35762, + "hast": 18146, + "hasta": 36623, + "hastings": 22035, + "hat": 3447, + "hat": 3801, + "hatch": 24202, + "hatch": 17809, + "hatchback": 42348, + "hatched": 42158, + "hate": 23546, + "hate": 3753, + "hated": 21298, + "hateful": 36418, + "hater": 36917, + "haters": 14027, + "hates": 14957, + "hatfield": 38448, + "hath": 27894, + "hath": 34416, + "hathaway": 31801, + "hati": 26045, + "hating": 25668, + "hatred": 19046, + "hats": 9812, + "hatt": 8747, + "hatton": 44861, + "hau": 5152, + "hauer": 48751, + "haul": 23743, + "haul": 12332, + "hauled": 46620, + "hauling": 43132, + "haun": 9676, + "haunt": 31039, + "haunted": 14944, + "haunting": 24034, + "haunts": 48035, + "haus": 41755, + "haus": 16478, + "hausen": 33338, + "hauser": 46586, + "haute": 28854, + "hav": 13443, + "hav": 20447, + "havan": 36304, + "havana": 23357, + "havas": 46261, + "have": 18053, + "have": 720, + "haven": 33074, + "haven": 3871, + "havent": 29130, + "haver": 27876, + "haves": 49088, + "havin": 31937, + "having": 1977, + "havoc": 24447, + "haw": 2788, + "haw": 26954, + "hawa": 6067, + "hawa": 46278, + "hawai": 15800, + "hawaii": 32413, + "hawaii": 8265, + "hawaiian": 17734, + "hawan": 27765, + "hawk": 14704, + "hawk": 8218, + "hawke": 38178, + "hawker": 39051, + "hawkeye": 38666, + "hawkeyes": 34266, + "hawking": 33437, + "hawkins": 19740, + "hawks": 44806, + "hawks": 5841, + "hawthorn": 45372, + "hawthorne": 36730, + "hay": 4871, + "hay": 11367, + "haya": 41325, + "hayat": 49360, + "hayden": 19806, + "haydn": 48207, + "haye": 36583, + "hayes": 13555, + "hayley": 39986, + "hayley": 22204, + "haynes": 30496, + "hays": 41524, + "hayward": 29400, + "haz": 5040, + "haz": 39921, + "hazard": 26174, + "hazard": 15178, + "hazardous": 27102, + "hazards": 30639, + "haze": 22785, + "hazel": 19838, + "hazel": 21882, + "hazelnut": 35816, + "hazi": 22740, + "hazmat": 48887, + "hazrat": 45775, + "hazy": 32655, + "hb": 6854, + "hb": 12576, + "hbcu": 40008, + "hbd": 25277, + "hbd": 13594, + "hbo": 15252, + "hc": 15831, + "hc": 7821, + "hcs": 46850, + "hd": 11601, + "hd": 4414, + "hdd": 40508, + "hdmi": 33302, + "hdr": 28065, + "he": 651, + "he": 797, + "hea": 27150, + "hea": 32790, + "head": 1603, + "head": 1375, + "headache": 23849, + "headaches": 38025, + "headband": 28556, + "headed": 6153, + "header": 11077, + "heading": 4409, + "headless": 45219, + "headlights": 42422, + "headline": 10891, + "headliner": 38880, + "headlines": 14706, + "headlining": 26971, + "headphone": 37524, + "headphones": 14906, + "headquarters": 13041, + "heads": 5174, + "headset": 23883, + "headshot": 34890, + "heal": 1231, + "heal": 13833, + "healed": 31456, + "healer": 38328, + "healey": 38985, + "healing": 9295, + "heals": 32384, + "health": 2145, + "health": 1728, + "healthand": 43704, + "healthcare": 42500, + "healthcare": 6023, + "healthier": 18242, + "healthtech": 42694, + "healthy": 10330, + "healthy": 3782, + "healthye": 31532, + "healthyeating": 33761, + "healthyfood": 39996, + "healthylifestyle": 46254, + "healthyliving": 27293, + "healy": 34299, + "heap": 34781, + "heaps": 44446, + "hear": 2749, + "hear": 2584, + "heard": 4063, + "hearing": 46353, + "hearing": 5541, + "hearings": 33175, + "hearn": 36613, + "hears": 25395, + "heart": 4975, + "heart": 1936, + "heartbeat": 29154, + "heartbreak": 29281, + "heartbreaking": 21322, + "heartbroken": 35383, + "hearted": 21679, + "heartfelt": 22904, + "hearth": 31563, + "hearthstone": 34054, + "hearti": 29345, + "hearties": 44572, + "heartland": 31923, + "heartless": 47022, + "heartnews": 40426, + "hearts": 5516, + "heartw": 30002, + "heartwarming": 34080, + "hearty": 26994, + "heat": 12175, + "heat": 4403, + "heated": 17057, + "heater": 23246, + "heath": 12794, + "heath": 11719, + "heather": 20230, + "heather": 12470, + "heathrow": 24171, + "heating": 12478, + "heaton": 34557, + "heats": 36106, + "heatwave": 25726, + "heav": 2409, + "heaven": 15520, + "heaven": 5545, + "heavenly": 19117, + "heavens": 26026, + "heavier": 31253, + "heaviest": 33268, + "heavily": 14123, + "heavy": 12048, + "heavy": 4200, + "heavymetal": 39804, + "heavyweight": 17448, + "heb": 24700, + "heb": 34515, + "hebdo": 41817, + "hebrew": 27298, + "hebrides": 45121, + "hebron": 45725, + "hec": 18932, + "heck": 22985, + "heck": 14427, + "hectares": 44162, + "hectic": 37245, + "hector": 25852, + "hed": 18271, + "hedge": 16229, + "hedge": 20294, + "hedgehog": 21940, + "hedges": 41345, + "hee": 18364, + "hee": 15773, + "heechul": 42487, + "heed": 15118, + "heel": 33646, + "heel": 16861, + "heels": 10909, + "heem": 30061, + "heer": 40473, + "hef": 29473, + "heff": 48756, + "hefty": 48584, + "heg": 41995, + "heh": 25834, + "hehe": 48723, + "hehe": 10658, + "hehehe": 24138, + "hei": 6101, + "hei": 29051, + "heidel": 42927, + "heidelberg": 48445, + "heidi": 44860, + "heidi": 23867, + "heifer": 48219, + "heigh": 43883, + "height": 10788, + "heights": 8418, + "heim": 10931, + "heim": 9768, + "heimer": 39517, + "hein": 15487, + "hein": 43206, + "heine": 28742, + "heineken": 36874, + "heinrich": 47877, + "heinz": 32359, + "heir": 27083, + "heir": 34007, + "heirloom": 34232, + "heirs": 43834, + "heis": 21849, + "heisman": 34537, + "heist": 31035, + "heit": 37255, + "hel": 919, + "hel": 11579, + "hela": 48212, + "held": 4042, + "hele": 46129, + "helen": 17576, + "helen": 11291, + "helena": 23109, + "helene": 41591, + "helens": 45940, + "heli": 33874, + "heli": 40183, + "helicop": 10035, + "helicopter": 11956, + "helicopters": 26922, + "helium": 46505, + "helix": 35247, + "hell": 8410, + "hell": 4141, + "hella": 19800, + "hellboy": 48428, + "helle": 48600, + "helle": 46968, + "hellenic": 42544, + "heller": 44464, + "hello": 12887, + "hello": 3306, + "hells": 47989, + "helly": 48690, + "helm": 47970, + "helm": 19520, + "helmet": 11122, + "helmets": 21843, + "help": 8641, + "help": 1318, + "helped": 4845, + "helper": 29321, + "helpers": 36316, + "helpful": 12695, + "helping": 3875, + "helpless": 47638, + "helpline": 43101, + "helps": 5144, + "helsin": 17842, + "helsinki": 19626, + "hem": 20270, + "hem": 11148, + "hemi": 14256, + "hemi": 46856, + "heming": 30819, + "hemingway": 33470, + "hemisphere": 32767, + "hemmings": 34882, + "hemo": 43788, + "hemp": 28225, + "hemp": 18467, + "hems": 32451, + "hemsworth": 39428, + "hen": 2385, + "hen": 8047, + "hence": 23640, + "hend": 11560, + "hender": 49248, + "henderson": 14348, + "hendrick": 45296, + "hendricks": 37588, + "hendrix": 23605, + "henge": 33104, + "henley": 27853, + "henna": 39455, + "hennessy": 42667, + "henri": 19431, + "henri": 21610, + "henrik": 35772, + "henry": 16018, + "henry": 5508, + "hens": 31742, + "henson": 32935, + "hep": 17724, + "hep": 48791, + "hepat": 23767, + "hepatitis": 32169, + "hepburn": 26348, + "her": 1223, + "her": 899, + "hera": 38724, + "heral": 37809, + "herald": 27625, + "herald": 12851, + "herb": 26116, + "herb": 15302, + "herbal": 21868, + "herbali": 44087, + "herbalife": 48364, + "herbert": 19935, + "herbs": 17320, + "hercules": 26539, + "herd": 36142, + "herd": 18589, + "here": 9134, + "here": 763, + "hered": 47976, + "hereford": 35543, + "heres": 13566, + "hereto": 47673, + "heri": 31392, + "herit": 4720, + "heritag": 38273, + "heritage": 20962, + "heritage": 5455, + "herman": 31890, + "herman": 21568, + "hermann": 40942, + "hermes": 34563, + "hermi": 35265, + "hermione": 45502, + "hermit": 43953, + "hermitage": 47706, + "hermo": 40967, + "hermosa": 42531, + "hern": 30571, + "hern": 43576, + "hernandez": 17707, + "hero": 7338, + "hero": 3756, + "heroes": 38010, + "heroes": 5506, + "heroic": 24255, + "heroin": 23841, + "heroine": 27420, + "heron": 22593, + "heros": 37642, + "herr": 38537, + "herrera": 27755, + "herring": 30211, + "hers": 25359, + "herself": 9207, + "hersh": 20379, + "hershey": 29734, + "hert": 26744, + "hertfordshire": 41070, + "herts": 35784, + "herty": 23454, + "hertz": 49383, + "hes": 30553, + "hes": 12784, + "hesit": 23933, + "hesitate": 34967, + "hess": 41888, + "hester": 31105, + "het": 37527, + "het": 19678, + "hetero": 26405, + "heu": 20105, + "heughan": 32298, + "hew": 48141, + "hew": 43051, + "hewitt": 28871, + "hex": 16255, + "hex": 31241, + "hey": 10759, + "hey": 2189, + "hez": 34591, + "hezbollah": 37636, + "hf": 26606, + "hf": 20603, + "hfx": 47297, + "hg": 23986, + "hg": 26237, + "hgtv": 47657, + "hh": 3280, + "hh": 5180, + "hhh": 8281, + "hhhh": 19391, + "hhhh": 13121, + "hhhhh": 24246, + "hhhhhh": 37278, + "hhs": 27006, + "hi": 677, + "hi": 1883, + "hia": 20672, + "hiatus": 27823, + "hib": 15922, + "hiber": 38799, + "hibis": 36226, + "hibiscus": 36460, + "hibition": 24658, + "hibs": 42814, + "hic": 3549, + "hic": 38079, + "hick": 14813, + "hickman": 49148, + "hickory": 29905, + "hicks": 23429, + "hid": 15552, + "hid": 14451, + "hidalgo": 47464, + "hidden": 28305, + "hidden": 7029, + "hiddleston": 31444, + "hide": 17725, + "hide": 9379, + "hideous": 46588, + "hides": 30800, + "hiding": 11371, + "hie": 15763, + "hier": 23433, + "hier": 29913, + "hierarchy": 44442, + "hifi": 38168, + "hig": 38108, + "higgins": 21783, + "high": 1487, + "high": 1400, + "higher": 5321, + "highered": 27072, + "highest": 5317, + "highland": 32244, + "highland": 16062, + "highlander": 46251, + "highlanders": 40445, + "highlands": 16883, + "highlight": 8264, + "highlighted": 22252, + "highlighter": 45460, + "highlighting": 17344, + "highlights": 6173, + "highly": 5302, + "highness": 38694, + "highs": 15144, + "highschool": 23102, + "highway": 45344, + "highway": 7620, + "highways": 28007, + "higu": 39115, + "hihi": 36240, + "hii": 42315, + "hijab": 31407, + "hika": 41356, + "hikari": 44624, + "hike": 9404, + "hiked": 36471, + "hiker": 40947, + "hikers": 46090, + "hikes": 27076, + "hiking": 9118, + "hiko": 48708, + "hil": 3508, + "hil": 17927, + "hila": 38837, + "hilar": 37337, + "hilari": 7784, + "hilarious": 8358, + "hilariously": 43476, + "hilary": 45898, + "hilary": 25415, + "hilde": 45382, + "hill": 3671, + "hill": 2682, + "hillary": 13257, + "hillary": 7074, + "hillaryclinton": 15357, + "hilli": 32513, + "hills": 24178, + "hills": 5289, + "hillsborough": 32157, + "hillside": 37194, + "hilltop": 45858, + "hilly": 32483, + "hilton": 33621, + "hilton": 14012, + "him": 4128, + "him": 1269, + "himach": 29132, + "himachal": 35461, + "himalay": 17552, + "himalayan": 30318, + "himalayas": 32872, + "hime": 45892, + "himself": 4530, + "himss": 41730, + "hin": 1676, + "hin": 37930, + "hina": 40571, + "hinakhan": 45518, + "hinch": 49320, + "hind": 34460, + "hind": 23293, + "hindi": 14967, + "hinds": 47859, + "hindu": 17587, + "hindu": 12053, + "hinduism": 40592, + "hindus": 25701, + "hindustan": 46553, + "hines": 37462, + "hing": 37968, + "hini": 33564, + "hino": 45343, + "hint": 11868, + "hinton": 47165, + "hints": 20594, + "hio": 32897, + "hip": 11725, + "hip": 6584, + "hipho": 8819, + "hiphop": 26598, + "hiphop": 10914, + "hipp": 13607, + "hippie": 28637, + "hippo": 28398, + "hippo": 36729, + "hips": 30191, + "hipstamatic": 31002, + "hipster": 19987, + "hipsters": 48265, + "hir": 4959, + "hir": 14728, + "hira": 42577, + "hire": 32356, + "hire": 8243, + "hired": 17602, + "hires": 24133, + "hiring": 7835, + "hiro": 17396, + "hiro": 20588, + "hiroshima": 33867, + "hirsch": 46967, + "his": 15211, + "his": 787, + "hism": 23502, + "hispan": 16843, + "hispanic": 22676, + "hist": 21710, + "hist": 13779, + "histo": 33479, + "histor": 2993, + "historia": 46010, + "historian": 20697, + "historians": 35200, + "historic": 30195, + "historic": 5726, + "historical": 34154, + "historical": 8039, + "historically": 30445, + "histories": 34736, + "history": 11142, + "history": 1695, + "historymonth": 19356, + "historyof": 35905, + "hit": 5453, + "hit": 2341, + "hitch": 22937, + "hitch": 36203, + "hitler": 16518, + "hitman": 33290, + "hits": 4712, + "hitter": 23538, + "hitters": 39724, + "hitting": 7957, + "hiv": 44410, + "hiv": 11018, + "hive": 38162, + "hive": 18521, + "hiya": 42393, + "hk": 22648, + "hk": 12307, + "hl": 8297, + "hl": 5956, + "hle": 32389, + "hler": 35418, + "hm": 17913, + "hm": 7631, + "hmm": 13725, + "hmmm": 17032, + "hmmmm": 34598, + "hms": 14625, + "hmu": 21630, + "hmv": 49288, + "hn": 22905, + "hn": 7478, + "hns": 48412, + "ho": 606, + "ho": 2971, + "hoa": 37517, + "hoar": 31628, + "hoax": 33438, + "hob": 18212, + "hobart": 31646, + "hobb": 16175, + "hobbies": 36370, + "hobbit": 23207, + "hobbs": 34343, + "hobby": 41120, + "hobby": 17557, + "hobo": 34613, + "hobo": 41334, + "hoboken": 41568, + "hoc": 35880, + "hoch": 43772, + "hock": 34914, + "hock": 46574, + "hockey": 16499, + "hockey": 4111, + "hoco": 34771, + "hod": 31062, + "hodg": 23660, + "hodge": 40585, + "hodges": 35061, + "hodgson": 37044, + "hoe": 32502, + "hoe": 11262, + "hoek": 40073, + "hoes": 21164, + "hof": 20186, + "hof": 12789, + "hofer": 38654, + "hoff": 32860, + "hoff": 22751, + "hofficial": 41949, + "hoffman": 22026, + "hog": 12075, + "hog": 13255, + "hogan": 19757, + "hogg": 42005, + "hogs": 23242, + "hogwarts": 29168, + "hoh": 43947, + "hoi": 39295, + "hok": 26942, + "hok": 47167, + "hokies": 35168, + "hokkaido": 49145, + "hol": 1187, + "hol": 7349, + "hola": 28724, + "hold": 36496, + "hold": 3254, + "holden": 21869, + "holder": 7862, + "holders": 10074, + "holding": 5050, + "holdings": 24832, + "holds": 7286, + "hole": 47242, + "hole": 5341, + "holes": 11266, + "holi": 2093, + "holi": 21926, + "holic": 16348, + "holics": 29782, + "holiday": 13168, + "holiday": 2878, + "holidays": 5372, + "holiness": 37259, + "holistic": 26300, + "holl": 27699, + "holla": 26500, + "holland": 31608, + "holland": 9978, + "hollande": 47690, + "holler": 49047, + "holli": 24019, + "holliday": 41624, + "hollow": 41221, + "hollow": 16691, + "holloway": 29435, + "holly": 12731, + "holly": 11923, + "hollyo": 41525, + "hollyoaks": 43352, + "hollywood": 24655, + "hollywood": 5518, + "holm": 34758, + "holm": 12739, + "holme": 46149, + "holmes": 12756, + "holo": 10317, + "holocau": 14688, + "holocaust": 16476, + "hols": 33344, + "holt": 18868, + "holtz": 44743, + "holy": 13910, + "holy": 4874, + "hom": 906, + "hom": 47397, + "homa": 9557, + "homage": 17746, + "home": 2143, + "home": 1137, + "homebrew": 35046, + "homec": 33869, + "homecoming": 9008, + "homedecor": 15695, + "homedepot": 38707, + "homegrown": 32554, + "homeitems": 42972, + "homeland": 21633, + "homeless": 18403, + "homeless": 9661, + "homelessness": 19851, + "homemade": 7889, + "homeof": 48856, + "homeowner": 37267, + "homeowners": 29882, + "homepage": 29828, + "homer": 29307, + "homer": 16931, + "homers": 38333, + "homes": 19480, + "homes": 5416, + "homeschool": 40994, + "homestead": 32609, + "homeswee": 46298, + "hometown": 12238, + "homework": 12495, + "homicide": 21520, + "homie": 12540, + "homies": 18893, + "homme": 26193, + "homo": 18129, + "homo": 30504, + "homophobia": 37875, + "homophobic": 40975, + "homosexual": 44288, + "homosexuality": 46720, + "homs": 45413, + "hon": 1279, + "hon": 10296, + "honda": 8553, + "honduras": 29715, + "hone": 38640, + "honest": 7814, + "honest": 9602, + "honestly": 9155, + "honesty": 24939, + "honey": 9843, + "honey": 6406, + "honeycomb": 48583, + "honeymoon": 22527, + "hong": 12144, + "hong": 8598, + "hongkong": 16659, + "honi": 17918, + "honolulu": 28096, + "honor": 9206, + "honor": 3402, + "honorable": 19498, + "honorary": 15675, + "honore": 25868, + "honored": 5494, + "honoree": 38993, + "honorees": 43012, + "honoring": 10771, + "honors": 10248, + "honour": 8240, + "honourable": 29855, + "honoured": 11945, + "honouring": 37754, + "honours": 22558, + "hoo": 2300, + "hoo": 7920, + "hood": 18681, + "hood": 3222, + "hooded": 33631, + "hoodie": 13444, + "hoodies": 25974, + "hoods": 16664, + "hoof": 44555, + "hook": 30488, + "hook": 10395, + "hookah": 34214, + "hooked": 18138, + "hookem": 31465, + "hooker": 37891, + "hooking": 35240, + "hooks": 25068, + "hooligans": 48176, + "hoon": 21368, + "hooo": 44538, + "hoop": 31516, + "hoop": 19573, + "hooper": 35221, + "hoops": 9351, + "hoor": 22155, + "hooray": 24940, + "hoos": 46462, + "hoosier": 48886, + "hoosiers": 42780, + "hoot": 29164, + "hoover": 25691, + "hop": 10848, + "hop": 5833, + "hope": 5263, + "hope": 1683, + "hoped": 30628, + "hopeful": 21453, + "hopefully": 7602, + "hopeless": 35586, + "hopes": 10018, + "hoping": 7207, + "hopkins": 17821, + "hopp": 48839, + "hopped": 34220, + "hopper": 21748, + "hopping": 27606, + "hoppy": 38359, + "hops": 21137, + "hor": 1407, + "hor": 33847, + "hora": 26013, + "horace": 39282, + "horan": 26857, + "horde": 44947, + "hore": 15380, + "horiz": 8144, + "horizon": 17924, + "horizon": 11920, + "horizons": 29685, + "horizontal": 25775, + "hormon": 27096, + "hormone": 31283, + "hormones": 35162, + "horn": 15771, + "horn": 9607, + "horne": 38143, + "horned": 34526, + "hornet": 28739, + "hornets": 20124, + "horns": 22109, + "horny": 32622, + "horo": 21500, + "horoscope": 38453, + "horowitz": 44669, + "horri": 8656, + "horrible": 13726, + "horribly": 45484, + "horrific": 25314, + "horrifying": 38901, + "horror": 13787, + "horror": 5032, + "horrormovies": 46682, + "horrors": 33321, + "horse": 8562, + "horse": 4558, + "horseback": 43673, + "horseman": 48885, + "horsepower": 36882, + "horser": 23096, + "horseracing": 30693, + "horses": 8809, + "horseshoe": 29242, + "horst": 37182, + "hort": 19482, + "horticul": 27141, + "horticulture": 39998, + "horton": 25945, + "hortons": 38422, + "horus": 29794, + "hos": 44320, + "hos": 25008, + "hosa": 44618, + "hose": 19662, + "hoseok": 38817, + "hosp": 2847, + "hosp": 37853, + "hospice": 20533, + "hospit": 7180, + "hospital": 29399, + "hospital": 3851, + "hospitality": 11657, + "hospitalized": 36915, + "hospitals": 13816, + "host": 17403, + "host": 3953, + "hostage": 26119, + "hoste": 31700, + "hosted": 6017, + "hostel": 27225, + "hostess": 39692, + "hostile": 28074, + "hosting": 4857, + "hosts": 8718, + "hot": 2851, + "hot": 2069, + "hota": 43289, + "hotdog": 43758, + "hotel": 14591, + "hotel": 2738, + "hotels": 8654, + "hotline": 30516, + "hotmail": 46427, + "hotness": 39803, + "hotra": 27109, + "hotro": 47823, + "hotspot": 36606, + "hotspur": 35176, + "hotter": 23591, + "hottest": 8279, + "hottie": 22804, + "hotties": 46027, + "hou": 1011, + "hou": 10122, + "hough": 44529, + "houghton": 36133, + "houn": 39273, + "houn": 33607, + "hound": 33996, + "hound": 13561, + "hounds": 21178, + "hounews": 48373, + "hour": 14930, + "hour": 2232, + "hourly": 30918, + "hours": 2382, + "house": 4107, + "house": 1212, + "housed": 37518, + "household": 12412, + "households": 27167, + "housel": 48685, + "housemusic": 28468, + "houseof": 19928, + "houses": 7791, + "housewives": 38523, + "housing": 32924, + "housing": 5734, + "houston": 16564, + "houston": 5663, + "hov": 40291, + "hove": 29674, + "hoven": 35559, + "hover": 36252, + "hover": 49016, + "hovering": 43437, + "how": 7470, + "how": 829, + "howar": 37672, + "howard": 25447, + "howard": 7632, + "howdy": 42216, + "howe": 8179, + "howe": 24614, + "howell": 25297, + "hower": 32920, + "however": 8467, + "howi": 47883, + "howie": 42939, + "howl": 40332, + "howling": 41771, + "howto": 38191, + "howto": 44060, + "hoy": 39625, + "hoy": 13278, + "hoya": 40978, + "hp": 23753, + "hp": 6371, + "hpa": 30983, + "hpc": 39936, + "hpe": 33787, + "hpv": 45765, + "hq": 33571, + "hq": 4693, + "hr": 4810, + "hr": 4086, + "hra": 21320, + "hra": 17212, + "hrc": 18139, + "hrh": 29103, + "hri": 21068, + "hrithik": 45371, + "hrs": 7157, + "hru": 24127, + "hrw": 25064, + "hs": 9343, + "hs": 2466, + "hsbc": 31508, + "hsc": 43510, + "hse": 34057, + "hsfb": 29539, + "hsv": 47311, + "ht": 11123, + "ht": 7801, + "hta": 23452, + "hta": 49384, + "htafc": 42821, + "htc": 48942, + "htc": 17635, + "html": 18231, + "hts": 43710, + "htt": 10620, + "http": 15066, + "https": 30901, + "httr": 49372, + "httweets": 43198, + "hu": 845, + "hu": 5949, + "hua": 22138, + "huan": 41405, + "huang": 32013, + "huar": 46916, + "huawe": 17709, + "huawei": 21128, + "hub": 18775, + "hub": 7028, + "hubb": 23183, + "hubbard": 33288, + "hubble": 30421, + "hubby": 16947, + "hubert": 40699, + "hubs": 29327, + "huck": 22909, + "huckabee": 43666, + "hud": 7169, + "hud": 28563, + "hudder": 22629, + "huddersfield": 24220, + "huddle": 33435, + "hudson": 25873, + "hudson": 11260, + "hue": 48380, + "hue": 21465, + "hues": 38003, + "huey": 39663, + "huff": 18746, + "huff": 44999, + "huffpost": 45887, + "hug": 40790, + "hug": 10359, + "huge": 2699, + "hugely": 24648, + "hugged": 41333, + "hugging": 27058, + "hugh": 8723, + "hugh": 15385, + "hughes": 11418, + "hugo": 43935, + "hugo": 17132, + "hugs": 14248, + "huh": 13348, + "huhu": 32134, + "hui": 29978, + "hul": 7911, + "hula": 40145, + "hulk": 17637, + "hull": 25154, + "hull": 10375, + "hulu": 24666, + "hum": 5823, + "hum": 16283, + "human": 3175, + "human": 2751, + "humane": 20220, + "humanitarian": 14170, + "humanities": 24949, + "humanity": 9420, + "humanright": 44385, + "humanrights": 14148, + "humans": 8324, + "humb": 9988, + "humber": 30602, + "humber": 38063, + "humble": 38703, + "humble": 10889, + "humbled": 19682, + "humbling": 39757, + "humbold": 24739, + "humboldt": 31389, + "hume": 38197, + "humid": 14778, + "humid": 27447, + "humidi": 47666, + "humidity": 15469, + "humil": 27205, + "humili": 25332, + "humility": 28535, + "humming": 26515, + "hummingbird": 33072, + "hummus": 31785, + "humor": 29369, + "humor": 11186, + "humorous": 38173, + "humour": 19161, + "hump": 16673, + "hump": 24529, + "humpback": 47662, + "humpday": 27693, + "humph": 19767, + "humphrey": 31549, + "hun": 1616, + "hun": 10795, + "hundre": 8505, + "hundred": 11898, + "hundreds": 8879, + "hung": 13825, + "hungar": 19420, + "hungarian": 23325, + "hungary": 17232, + "hunger": 25565, + "hunger": 10184, + "hungergames": 47507, + "hungover": 41110, + "hungry": 44845, + "hungry": 8451, + "hunk": 33912, + "hunt": 16498, + "hunt": 5774, + "hunted": 37373, + "hunter": 16531, + "hunter": 6099, + "hunters": 16115, + "hunting": 27830, + "hunting": 7507, + "huntington": 23521, + "hunts": 34041, + "huntsville": 34544, + "hur": 2305, + "hur": 34523, + "hurd": 44915, + "hurdle": 27486, + "hurdles": 25440, + "huri": 42486, + "hurley": 30166, + "hurling": 24738, + "huron": 36147, + "hurrah": 40599, + "hurric": 6543, + "hurrican": 36105, + "hurricane": 24051, + "hurricane": 8782, + "hurricanes": 22357, + "hurry": 10921, + "hurst": 44742, + "hurst": 11760, + "hurt": 7413, + "hurting": 24017, + "hurts": 13059, + "hus": 5111, + "hus": 35853, + "husband": 6179, + "husbands": 33612, + "hush": 28728, + "husk": 19246, + "huskers": 26946, + "huskies": 20988, + "husky": 20421, + "huss": 13733, + "hussain": 17940, + "hussein": 31336, + "hust": 27279, + "hustle": 15709, + "huston": 46480, + "hut": 20924, + "hut": 16503, + "hutch": 31018, + "hutch": 33203, + "hutchinson": 35721, + "hutto": 27662, + "hutton": 38321, + "hv": 17209, + "hv": 18593, + "hvac": 27492, + "hw": 27491, + "hw": 18876, + "hwa": 32352, + "hwan": 44390, + "hwang": 46775, + "hwy": 13812, + "hy": 1441, + "hy": 17827, + "hya": 31600, + "hyacin": 47263, + "hyatt": 44856, + "hyatt": 25146, + "hybri": 9084, + "hybrid": 10156, + "hyd": 42382, + "hyde": 46484, + "hyde": 16343, + "hyder": 13960, + "hyderabad": 14801, + "hydr": 8031, + "hydra": 44414, + "hydra": 40420, + "hydrange": 43298, + "hydrate": 29628, + "hydrated": 23300, + "hydrating": 47653, + "hydration": 24174, + "hydrau": 26017, + "hydraulic": 26189, + "hydro": 8368, + "hydro": 22595, + "hydrogen": 20974, + "hye": 32724, + "hye": 25792, + "hygi": 16277, + "hygiene": 19591, + "hymn": 41350, + "hyo": 38960, + "hyo": 35078, + "hyp": 16964, + "hype": 30353, + "hype": 11111, + "hyped": 22507, + "hyper": 7997, + "hyper": 22146, + "hypertension": 40698, + "hypno": 23355, + "hypnosis": 48138, + "hypnoti": 40440, + "hypo": 10252, + "hypocr": 30711, + "hypocri": 25606, + "hypocrisy": 26296, + "hypocrite": 44125, + "hypothe": 46966, + "hypothesis": 44956, + "hyster": 24235, + "hysteria": 45965, + "hysterical": 48627, + "hyuk": 20452, + "hyun": 11831, + "hyun": 8589, + "hyundai": 17094, + "hyung": 46901, + "hyung": 16551, + "hz": 32533, + "i": 72, + "i": 328, + "ia": 12486, + "ia": 1073, + "iac": 32838, + "iac": 44063, + "iaf": 40789, + "iah": 35052, + "iain": 30103, + "ial": 11530, + "ial": 1974, + "ials": 20940, + "iam": 3579, + "iam": 11415, + "iambic": 43668, + "iambicpent": 43891, + "iamsrk": 15103, + "ian": 7723, + "ian": 1800, + "ians": 6451, + "iansomerhalder": 47077, + "iart": 18413, + "iartg": 18669, + "ias": 32303, + "ias": 14620, + "ib": 3962, + "ib": 13554, + "iba": 39763, + "ibadan": 44691, + "iban": 47145, + "ibc": 49014, + "ibd": 40732, + "iber": 23814, + "ibi": 12337, + "ibis": 47048, + "ibiza": 13853, + "ible": 37792, + "ibles": 44102, + "ibm": 23415, + "ibm": 13918, + "ibn": 25729, + "ibooks": 46887, + "ibra": 15476, + "ibrahi": 40350, + "ibrahim": 20816, + "ibrox": 46883, + "ibs": 41993, + "ibu": 43587, + "ibu": 46117, + "ic": 535, + "ic": 1029, + "ica": 2576, + "icago": 37492, + "ical": 6082, + "ical": 1110, + "ically": 3161, + "icals": 13999, + "ican": 17653, + "ican": 5246, + "icans": 20511, + "icar": 37211, + "ication": 21629, + "icc": 12945, + "ice": 2739, + "ice": 733, + "iceberg": 33662, + "icec": 13636, + "icecream": 21334, + "iced": 8049, + "icelan": 34114, + "iceland": 46716, + "iceland": 11935, + "icelandic": 34705, + "ices": 1931, + "ich": 5333, + "ich": 1232, + "icha": 31453, + "iche": 28972, + "iche": 21143, + "ichi": 21669, + "ichi": 14647, + "ichick": 45022, + "ichiro": 43787, + "ici": 948, + "ici": 22189, + "icia": 11774, + "icial": 17543, + "icial": 6397, + "ician": 40522, + "ician": 5374, + "icians": 6264, + "iciary": 21329, + "icic": 46006, + "icide": 6558, + "icides": 28253, + "icing": 7676, + "icio": 24207, + "icion": 45905, + "icious": 3325, + "icist": 21165, + "icists": 42171, + "icity": 7243, + "ick": 1168, + "ick": 1068, + "icked": 39799, + "icker": 40357, + "ickers": 30701, + "icki": 35468, + "icking": 6619, + "icks": 3727, + "icky": 11587, + "icn": 44516, + "ico": 13697, + "ico": 3040, + "icom": 17693, + "icom": 29796, + "icon": 13843, + "icon": 5646, + "iconic": 6959, + "icons": 15553, + "icop": 9389, + "icos": 32002, + "ics": 1324, + "ict": 6349, + "icted": 36515, + "iction": 40560, + "icton": 36548, + "icu": 45118, + "icu": 30443, + "icular": 40660, + "icus": 31459, + "icy": 28780, + "icy": 3495, + "icymi": 5315, + "icz": 46387, + "id": 1568, + "id": 1014, + "ida": 11032, + "ida": 11600, + "idad": 22462, + "idaho": 48817, + "idaho": 15165, + "idal": 39684, + "idan": 17929, + "idc": 22386, + "ide": 1909, + "ide": 14104, + "idea": 3612, + "ideal": 8789, + "ideally": 48247, + "ideals": 45096, + "ideas": 4452, + "ident": 7113, + "identi": 6009, + "identical": 25587, + "identification": 23337, + "identified": 15217, + "identifies": 35712, + "identify": 10949, + "identifying": 23589, + "identities": 34292, + "identity": 8892, + "ideology": 25840, + "iders": 8980, + "ides": 31791, + "idf": 28987, + "idge": 35567, + "idh": 44325, + "idi": 9611, + "idi": 14264, + "idio": 15994, + "idiot": 14087, + "idiots": 20856, + "idk": 8972, + "idle": 34754, + "idlib": 36199, + "ido": 6763, + "ido": 29641, + "idol": 24866, + "idol": 8884, + "idols": 21398, + "idr": 10106, + "idri": 46435, + "idris": 41312, + "ids": 6111, + "idu": 28655, + "idy": 33058, + "idyl": 44879, + "idyllic": 46632, + "ie": 6789, + "ie": 1718, + "iec": 44773, + "ied": 10059, + "ieee": 39860, + "iel": 27875, + "iel": 22729, + "ience": 1542, + "ient": 13115, + "ier": 33173, + "ier": 5912, + "iers": 45060, + "ies": 27912, + "ies": 963, + "iest": 10818, + "if": 8063, + "if": 878, + "ifa": 37574, + "ifc": 36524, + "ife": 41172, + "ife": 19590, + "iff": 35753, + "ification": 35755, + "ified": 41403, + "ift": 31143, + "iftar": 35153, + "ifu": 41523, + "ify": 32807, + "ig": 1089, + "ig": 3072, + "iga": 16493, + "igan": 27468, + "igans": 25419, + "igbo": 44591, + "ige": 10806, + "igen": 33070, + "iger": 30758, + "iger": 20685, + "igers": 40755, + "igers": 48928, + "iggy": 46219, + "iggy": 27604, + "igh": 2712, + "igh": 5451, + "ight": 14571, + "ight": 897, + "ighton": 35292, + "igi": 21901, + "igle": 29912, + "iglesias": 39432, + "ign": 7303, + "ign": 2326, + "ignati": 37573, + "ignatius": 48318, + "igne": 45843, + "ignite": 25210, + "ignition": 36115, + "igno": 15375, + "ignor": 7653, + "ignorance": 22735, + "ignorant": 26933, + "ignore": 12304, + "ignored": 20428, + "ignores": 40129, + "ignoring": 23969, + "igor": 33024, + "igs": 31344, + "igu": 21279, + "ih": 12162, + "ih": 34135, + "ihear": 13043, + "iheart": 30332, + "iheartawards": 18811, + "iheartradio": 25934, + "ihop": 45511, + "ihri": 39108, + "ihrithik": 39326, + "ii": 5103, + "ii": 2329, + "iii": 46236, + "iii": 6572, + "iiii": 20133, + "iiii": 45393, + "iiot": 30704, + "iit": 39330, + "iit": 33238, + "ij": 7337, + "ija": 42802, + "ik": 3903, + "ik": 10177, + "ika": 18188, + "ike": 12329, + "ike": 19696, + "ikea": 20528, + "iker": 38653, + "ikh": 44655, + "ikh": 12758, + "iklan": 32028, + "iklan": 29584, + "iko": 35659, + "iko": 39272, + "ikon": 38543, + "ikon": 19156, + "iku": 17780, + "il": 543, + "il": 958, + "ila": 4344, + "ilah": 32211, + "ilan": 13889, + "ilan": 28076, + "iland": 20957, + "ilation": 16180, + "ilay": 45093, + "ild": 22278, + "ild": 17164, + "ile": 18398, + "ile": 989, + "iled": 3358, + "iler": 22446, + "iler": 3615, + "ilers": 8975, + "iles": 42274, + "ili": 2076, + "ili": 19601, + "ilia": 14855, + "ilian": 10272, + "iliary": 32585, + "ilife": 42835, + "ilike": 44989, + "ilinan": 48497, + "iling": 3299, + "ilio": 47256, + "ilion": 12561, + "ilis": 43442, + "ilit": 11178, + "ilities": 5446, + "ility": 1787, + "ilive": 26478, + "ill": 828, + "ill": 660, + "illa": 8877, + "illa": 3043, + "illac": 17218, + "illage": 48922, + "illard": 21920, + "illary": 33667, + "illas": 23404, + "ille": 18213, + "ille": 5559, + "illed": 2527, + "illeg": 35808, + "illegal": 7983, + "illegally": 24466, + "illegals": 40490, + "iller": 23341, + "iller": 2956, + "illers": 30547, + "illery": 14514, + "illes": 20037, + "illi": 1086, + "illi": 25187, + "illia": 48776, + "illiams": 30301, + "illian": 48775, + "illian": 17355, + "illic": 37152, + "illicit": 40998, + "illie": 26083, + "illin": 35868, + "illing": 2803, + "illini": 28957, + "illino": 8920, + "illinois": 9414, + "illion": 35542, + "illion": 2035, + "illness": 11145, + "illnesses": 33861, + "illo": 34153, + "illo": 7588, + "illon": 20516, + "ills": 1900, + "illu": 3025, + "illumin": 11446, + "illuminate": 43261, + "illuminated": 28814, + "illuminati": 34551, + "illuminating": 46601, + "illumination": 43680, + "illus": 41386, + "illusion": 20318, + "illusions": 47429, + "illustr": 6268, + "illustrate": 37468, + "illustrated": 13151, + "illustrates": 38129, + "illustrating": 43322, + "illustration": 6052, + "illustrations": 17852, + "illustrator": 16649, + "illustri": 43116, + "illustrious": 44304, + "illy": 11707, + "illy": 9532, + "ilm": 36326, + "ilo": 4220, + "ilo": 14835, + "ilove": 7183, + "ilove": 32914, + "iloveart": 41114, + "ilovemy": 28863, + "iloveyou": 28829, + "ils": 1543, + "ilt": 25334, + "ilton": 28494, + "ilu": 27337, + "ilwx": 43777, + "ily": 4881, + "ily": 1026, + "ilya": 33377, + "ilysm": 29228, + "im": 732, + "im": 1496, + "ima": 2414, + "ima": 6432, + "imac": 40675, + "imacele": 47281, + "imag": 2316, + "image": 24101, + "image": 2867, + "imagery": 22828, + "images": 4952, + "imagin": 18178, + "imaginary": 30417, + "imagination": 13783, + "imaginative": 47233, + "imagine": 35752, + "imagine": 4826, + "imagined": 18478, + "imagines": 47379, + "imaging": 14231, + "imagining": 27384, + "imam": 37552, + "imam": 19024, + "iman": 45684, + "iman": 16247, + "imation": 44566, + "imax": 32066, + "imc": 45616, + "imdanielpadilla": 36357, + "imdb": 30407, + "ime": 44937, + "ime": 31151, + "imel": 31594, + "iment": 37157, + "imer": 21802, + "imes": 47744, + "imf": 28403, + "img": 24157, + "imi": 23559, + "imin": 23942, + "imit": 23462, + "imitation": 41630, + "imma": 19487, + "immac": 25085, + "immaculate": 29649, + "immature": 45531, + "immedi": 7366, + "immediate": 14440, + "immediately": 10108, + "immen": 17278, + "immense": 22722, + "immensely": 35013, + "immer": 13954, + "immerse": 46240, + "immersion": 31861, + "immersive": 27521, + "immigr": 5851, + "immigrant": 16474, + "immigrants": 14460, + "immigration": 9588, + "imminent": 27299, + "immort": 39244, + "immortal": 24717, + "immun": 8961, + "immune": 15606, + "immuni": 44571, + "immunity": 26254, + "immuno": 24361, + "immunology": 44483, + "immunotherapy": 39185, + "imo": 26349, + "imo": 13738, + "imp": 3335, + "imp": 31037, + "impac": 7573, + "impact": 33036, + "impact": 3844, + "impacted": 21424, + "impactful": 41631, + "impacting": 29359, + "impacts": 15069, + "impair": 36451, + "impaired": 28028, + "impairment": 44501, + "impala": 36641, + "impe": 23612, + "impeach": 16874, + "impeach": 43497, + "impeachment": 32979, + "impeachtrump": 38006, + "impecc": 34511, + "impeccable": 40111, + "impending": 34486, + "imper": 7727, + "imperative": 39833, + "imperfect": 46034, + "imperi": 30911, + "imperial": 32425, + "imperial": 12361, + "imperialism": 48855, + "imperson": 25551, + "implant": 33106, + "implants": 32202, + "imple": 7423, + "implement": 17966, + "implementation": 15102, + "implemented": 24315, + "implementing": 22862, + "implic": 15269, + "implications": 19229, + "implo": 40337, + "impo": 45704, + "import": 2336, + "import": 16294, + "importance": 6821, + "important": 2829, + "importantly": 21580, + "imported": 28798, + "imports": 25286, + "impose": 35879, + "imposed": 25871, + "imposing": 42289, + "impossible": 9815, + "impre": 3763, + "impress": 20015, + "impressed": 9689, + "impression": 14468, + "impressionism": 36114, + "impressionist": 44904, + "impressions": 22276, + "impressive": 6634, + "imprint": 43863, + "imprison": 22141, + "imprisoned": 32999, + "imprisonment": 39024, + "impro": 2531, + "impromp": 28100, + "impromptu": 28611, + "improv": 22868, + "improve": 4971, + "improved": 9446, + "improvement": 10790, + "improvements": 16320, + "improves": 18035, + "improving": 10381, + "improvis": 32343, + "improvised": 40886, + "impulse": 29683, + "impy": 42690, + "imran": 19647, + "imran": 19212, + "imrankhan": 25956, + "imrankhanpti": 26688, + "ims": 17800, + "imsa": 37262, + "imv": 35731, + "imvkohli": 37136, + "imwith": 26822, + "imwithher": 32651, + "in": 512, + "in": 530, + "ina": 18026, + "ina": 1366, + "inability": 47517, + "inaccurate": 49192, + "inaction": 41916, + "inactive": 49274, + "inadequate": 43403, + "inak": 46549, + "inal": 19178, + "inals": 26438, + "inan": 26204, + "inappropriate": 26722, + "inari": 48620, + "inary": 11337, + "inas": 36731, + "inas": 12362, + "inated": 38530, + "ination": 4706, + "inau": 10832, + "inaugu": 11309, + "inaugur": 11448, + "inaugural": 11340, + "inaugurated": 29011, + "inauguration": 16805, + "inbound": 24420, + "inbox": 18683, + "inc": 14570, + "inc": 4438, + "incan": 45964, + "incar": 18070, + "incarcer": 26334, + "incarcerated": 49178, + "incarceration": 39887, + "incase": 30463, + "ince": 44303, + "incen": 13259, + "incense": 35059, + "incentive": 29024, + "incentives": 29813, + "inception": 36653, + "inch": 6523, + "incheon": 30645, + "inches": 10809, + "inci": 5747, + "incidence": 43371, + "incident": 10103, + "incidents": 22120, + "incindia": 26161, + "inciner": 46434, + "incl": 27857, + "incl": 13338, + "inclined": 45470, + "inclu": 1738, + "include": 5942, + "included": 7414, + "includes": 6197, + "including": 2814, + "inclusion": 12079, + "inclusive": 13393, + "income": 8044, + "incoming": 15416, + "incomparable": 36027, + "incompetent": 45069, + "incomplete": 34040, + "incon": 42372, + "inconvenience": 40563, + "incorpor": 19335, + "incorporate": 34168, + "incorporated": 29494, + "incorporating": 40303, + "incorrect": 31872, + "incre": 1870, + "increase": 5230, + "increased": 9156, + "increases": 13797, + "increasing": 10270, + "increasingly": 16106, + "incredi": 2883, + "incredible": 22128, + "incredible": 3457, + "incredibleindia": 24680, + "incredibles": 48641, + "incredibly": 9513, + "incu": 38830, + "incub": 24587, + "incubator": 35736, + "incumb": 32246, + "incumbent": 38038, + "incur": 42356, + "ind": 5386, + "ind": 4655, + "inda": 15710, + "inde": 2645, + "indeed": 10031, + "indefin": 29501, + "indefinitely": 43750, + "independ": 4147, + "independence": 23117, + "independence": 7955, + "independenceday": 25971, + "independent": 33844, + "independent": 7088, + "independently": 39831, + "inder": 29225, + "index": 35209, + "index": 9458, + "indhoven": 44229, + "indi": 1098, + "indi": 46536, + "india": 27067, + "india": 1762, + "indian": 7685, + "indian": 3606, + "indiana": 8615, + "indianapolis": 17196, + "indianfootball": 45979, + "indians": 10271, + "indic": 7136, + "indicate": 26679, + "indicated": 39416, + "indicates": 29412, + "indication": 38539, + "indicator": 24776, + "indicators": 30054, + "indicted": 34992, + "indictment": 42278, + "indie": 5260, + "indie": 9383, + "indiedev": 10863, + "indiefilm": 22588, + "indiegame": 17969, + "indiegamedev": 40466, + "indiegames": 35864, + "indiegogo": 38057, + "indies": 23618, + "indiffe": 41372, + "indigen": 8348, + "indigenous": 9303, + "indigo": 21002, + "indira": 43887, + "indirec": 26398, + "indirect": 35416, + "indivi": 5649, + "individu": 9574, + "individual": 8512, + "individually": 33782, + "individuals": 11990, + "indo": 26303, + "indo": 18297, + "indom": 42926, + "indone": 6180, + "indonesia": 7229, + "indonesian": 19593, + "indoor": 44478, + "indoor": 9546, + "indoors": 22973, + "indore": 46143, + "indu": 2298, + "induc": 7973, + "induced": 24103, + "inducted": 20596, + "inductee": 39558, + "inductees": 44796, + "induction": 18338, + "indul": 19402, + "indulg": 28388, + "indulge": 24851, + "indulgence": 40856, + "indulgent": 49147, + "industri": 5082, + "industrial": 30853, + "industrial": 7520, + "industries": 11700, + "industry": 47407, + "industry": 3318, + "indv": 16942, + "indy": 9821, + "indy": 10098, + "indycar": 20484, + "indyref": 22569, + "ine": 855, + "ine": 715, + "ineau": 38122, + "inec": 45214, + "ined": 2038, + "inee": 43252, + "inee": 7986, + "inees": 13056, + "ineffe": 47202, + "inely": 18234, + "inem": 48876, + "inema": 29232, + "inen": 44365, + "inequalities": 45507, + "inequality": 17372, + "iner": 17438, + "iner": 5155, + "iners": 41863, + "ines": 2137, + "inese": 35966, + "iness": 1463, + "inet": 8121, + "inette": 38911, + "inev": 19527, + "inevit": 45871, + "inevitable": 25004, + "inews": 24300, + "inexpensive": 38614, + "iney": 30254, + "inez": 12700, + "inf": 1529, + "inf": 35241, + "infamous": 18688, + "infan": 17219, + "infant": 19192, + "infantry": 21655, + "infants": 34726, + "infe": 7164, + "infec": 26088, + "infected": 26136, + "infection": 14774, + "infections": 22227, + "infectious": 29157, + "infeld": 25035, + "infer": 16258, + "inferno": 31290, + "infertility": 40701, + "infield": 48933, + "infiltr": 28683, + "infin": 6246, + "infinite": 12748, + "infiniti": 34644, + "infinity": 34863, + "infinity": 12895, + "infl": 7627, + "inflam": 16080, + "inflammation": 24893, + "inflammatory": 26831, + "inflatable": 30135, + "inflation": 17497, + "inflicted": 48188, + "influ": 4835, + "influen": 13229, + "influence": 9199, + "influenced": 21183, + "influencer": 25013, + "influencers": 29891, + "influences": 24926, + "influencing": 45126, + "influential": 17553, + "influenza": 39897, + "info": 5680, + "info": 2222, + "infographic": 10076, + "infographics": 33172, + "infor": 31773, + "inform": 10241, + "inform": 19449, + "informal": 25705, + "informat": 29625, + "informatics": 35685, + "information": 3204, + "informative": 19364, + "informed": 13876, + "informing": 45388, + "informs": 48440, + "infosec": 17863, + "infr": 29718, + "infra": 7312, + "infra": 45877, + "infrared": 22867, + "infrastructure": 9034, + "infringe": 44882, + "infringement": 48712, + "infront": 37668, + "infu": 15048, + "infuri": 48461, + "infused": 21461, + "infusion": 43464, + "ing": 653, + "ing": 519, + "inga": 15233, + "ingco": 40444, + "ingday": 16561, + "ingdon": 38731, + "inge": 11790, + "inge": 7071, + "inged": 30046, + "ingen": 19088, + "ingeni": 36884, + "inger": 33883, + "inger": 3541, + "ingfor": 33430, + "ingh": 9170, + "ingh": 30495, + "ingham": 24497, + "ingham": 4291, + "inghamshire": 39289, + "inghour": 42728, + "inging": 4066, + "ingl": 45662, + "ingle": 22228, + "ingle": 17005, + "ingles": 24490, + "ingley": 44428, + "inglis": 46327, + "ingly": 4796, + "ingnow": 34766, + "ingo": 30175, + "ingo": 9012, + "ingra": 45165, + "ingrad": 44124, + "ingram": 26998, + "ingredi": 9272, + "ingredient": 19799, + "ingredients": 11788, + "ingrid": 33496, + "ings": 895, + "ingthe": 20170, + "ingtips": 39373, + "ington": 11846, + "ington": 2156, + "ingu": 8714, + "ingual": 22795, + "ingue": 36838, + "ingui": 12788, + "inguish": 36146, + "inha": 32612, + "inhabit": 36189, + "inhabitants": 44968, + "inhal": 30786, + "inhe": 32617, + "inher": 24611, + "inherent": 47327, + "inherit": 34322, + "inheritance": 39341, + "inherited": 39111, + "inhi": 25557, + "inhibit": 32196, + "inho": 12984, + "ini": 6154, + "ini": 3581, + "inian": 36638, + "inim": 38717, + "inindia": 34021, + "ining": 1389, + "inist": 30976, + "init": 42670, + "initi": 4580, + "initial": 13980, + "initially": 28123, + "initials": 48794, + "initiated": 27756, + "initiation": 41009, + "initiative": 8152, + "initiatives": 16549, + "inity": 22126, + "inj": 5112, + "injec": 13688, + "injection": 21438, + "inju": 5006, + "injured": 7505, + "injuries": 9481, + "injury": 6223, + "injustice": 20541, + "ink": 4547, + "ink": 967, + "inka": 40685, + "inked": 29356, + "inki": 46176, + "inkigayo": 47882, + "inking": 37586, + "inks": 20966, + "inktober": 9387, + "inland": 21943, + "inlet": 35161, + "inline": 45004, + "inlove": 28415, + "inmate": 32341, + "inmates": 28216, + "inmy": 42657, + "inn": 27260, + "inn": 5569, + "inna": 35088, + "inner": 24512, + "inner": 6955, + "inning": 4415, + "innings": 11580, + "innis": 44059, + "inno": 7961, + "innocence": 26383, + "innocent": 11241, + "innov": 2890, + "innovate": 24549, + "innovation": 33063, + "innovation": 4272, + "innovations": 18817, + "innovative": 8494, + "innovator": 34735, + "innovators": 27834, + "ino": 4211, + "ino": 2691, + "inoa": 25649, + "inos": 21828, + "inous": 47801, + "inox": 22698, + "input": 16952, + "inputs": 48763, + "inqu": 10628, + "inqui": 18527, + "inquirer": 45172, + "inquiries": 29469, + "inquiry": 15865, + "inquis": 31171, + "inr": 36325, + "ins": 12786, + "ins": 1041, + "insan": 7875, + "insane": 10260, + "insanely": 27846, + "insanity": 26645, + "inscribed": 49168, + "inscription": 41127, + "insec": 15744, + "insect": 21297, + "insects": 18714, + "insecure": 35112, + "insecurity": 36964, + "inser": 13830, + "insert": 18807, + "insi": 3453, + "inside": 19141, + "inside": 2912, + "insider": 13300, + "insiders": 32171, + "insig": 40503, + "insight": 8795, + "insightful": 20354, + "insights": 8729, + "insignia": 48864, + "insist": 35504, + "insisted": 40423, + "insists": 27255, + "inski": 32630, + "insky": 24607, + "insol": 42366, + "insom": 21755, + "insomni": 42040, + "insomnia": 30598, + "inson": 21007, + "insp": 1597, + "inspec": 7915, + "inspect": 40815, + "inspecting": 40565, + "inspection": 15142, + "inspections": 39513, + "inspector": 20514, + "inspir": 2573, + "inspiration": 4195, + "inspirational": 41936, + "inspirational": 9855, + "inspirations": 35093, + "inspire": 27901, + "inspire": 8583, + "inspired": 39849, + "inspired": 3516, + "inspires": 17245, + "inspiring": 41847, + "inspiring": 5705, + "inspo": 26897, + "inst": 1264, + "inst": 1581, + "insta": 22411, + "insta": 11694, + "instability": 41377, + "instac": 46678, + "instaf": 33800, + "instag": 14612, + "instagood": 23718, + "instagram": 27910, + "instagram": 2659, + "instal": 38805, + "install": 6940, + "install": 11168, + "installation": 9358, + "installations": 27909, + "installed": 8807, + "installing": 18301, + "installment": 25315, + "installs": 45568, + "instalment": 47766, + "instance": 34572, + "instant": 38810, + "instant": 10635, + "instantly": 17703, + "instap": 23758, + "instapic": 34378, + "instaweather": 43078, + "instaweatherpro": 43150, + "inste": 3571, + "instead": 4191, + "instein": 13421, + "instem": 27030, + "instin": 23382, + "instinct": 30544, + "institu": 4257, + "institute": 5861, + "institutes": 43674, + "institution": 18823, + "institutional": 27442, + "institutions": 15207, + "instore": 41679, + "instru": 4544, + "instruc": 19648, + "instruction": 19407, + "instructional": 31022, + "instructions": 17040, + "instructor": 16087, + "instructors": 31998, + "instrument": 42196, + "instrument": 15806, + "instrumental": 23041, + "instruments": 14793, + "instyle": 41321, + "insu": 8805, + "insul": 9615, + "insulated": 42051, + "insulation": 28194, + "insulin": 29311, + "insult": 26673, + "insulting": 39646, + "insults": 40451, + "insur": 5024, + "insurance": 5870, + "insured": 31321, + "insurers": 43142, + "insurtech": 28716, + "int": 1828, + "int": 1207, + "inta": 38314, + "intact": 26870, + "intake": 19539, + "intan": 47695, + "inte": 1598, + "inte": 41900, + "intech": 26504, + "inted": 6147, + "integr": 5151, + "integral": 27018, + "integrate": 25735, + "integrated": 12797, + "integrating": 31555, + "integration": 12583, + "integrity": 14791, + "intel": 11778, + "intel": 11426, + "intellec": 13281, + "intellect": 47828, + "intellectu": 31966, + "intellectual": 18069, + "intelli": 5324, + "intellig": 5632, + "intelligence": 6846, + "intelligent": 14063, + "inten": 2967, + "intend": 36674, + "intended": 16812, + "intense": 10258, + "intensi": 22928, + "intensity": 19956, + "intensive": 21049, + "intent": 18881, + "intention": 26786, + "intentional": 29536, + "intentionally": 31215, + "intentions": 26710, + "inter": 1006, + "inter": 10093, + "interact": 21736, + "interacting": 35045, + "interaction": 17650, + "interactions": 22162, + "interactive": 9456, + "intercep": 23676, + "interception": 48762, + "interceptions": 45313, + "interchange": 34222, + "intercontinental": 31983, + "interdisciplinary": 38132, + "intere": 2008, + "interest": 5095, + "interested": 4620, + "interesting": 3628, + "interests": 16425, + "interface": 18753, + "interfaith": 38399, + "interference": 29099, + "interim": 19509, + "interior": 10700, + "interior": 7305, + "interiordesign": 12902, + "interiors": 14836, + "intermedi": 20246, + "intermediate": 24304, + "intermission": 44805, + "intermitt": 44946, + "intern": 9976, + "intern": 14068, + "internal": 11285, + "internally": 41134, + "internation": 42534, + "international": 8566, + "international": 2436, + "internationaldayof": 41518, + "internationally": 24059, + "internationalwomensday": 17682, + "interne": 32713, + "internet": 30180, + "internet": 4757, + "internetof": 44449, + "internetofthings": 45925, + "interns": 19902, + "internship": 16661, + "internships": 39410, + "interoper": 45754, + "interpre": 11162, + "interpret": 49154, + "interpret": 40459, + "interpretation": 20652, + "interpreted": 42157, + "interpreting": 46525, + "interro": 29548, + "interrup": 21609, + "interrupt": 48449, + "interrupted": 30288, + "intersec": 45246, + "intersection": 19210, + "interstate": 21963, + "interstellar": 41506, + "interval": 36032, + "intervals": 44884, + "interven": 18245, + "intervention": 16804, + "interventions": 28848, + "interview": 2885, + "interviewed": 11688, + "interviewing": 16399, + "interviews": 9910, + "intestin": 37938, + "intestinal": 38896, + "inthe": 7486, + "inti": 14459, + "intim": 38832, + "intimacy": 46430, + "intimate": 16382, + "intimid": 24041, + "intimidating": 44405, + "intimidation": 49258, + "inting": 15571, + "intl": 38186, + "intl": 14224, + "intment": 9020, + "intments": 21420, + "into": 35235, + "into": 1095, + "intoler": 28534, + "intolerance": 37808, + "intothe": 38511, + "intra": 20922, + "intrac": 46195, + "intram": 40956, + "intre": 29397, + "intrepid": 39127, + "intri": 15421, + "intric": 23763, + "intricate": 29616, + "intrigu": 18856, + "intrigue": 45140, + "intrigued": 40034, + "intriguing": 24334, + "intrin": 45181, + "intro": 2999, + "intro": 13224, + "introduc": 3621, + "introduce": 9813, + "introduced": 10446, + "introduces": 12933, + "introducing": 6256, + "introduction": 11812, + "introductory": 38121, + "intru": 22949, + "ints": 2514, + "intu": 17225, + "intuition": 40897, + "intuitive": 35224, + "inu": 21131, + "inuit": 41250, + "inus": 45857, + "inv": 2279, + "inv": 43786, + "inva": 10084, + "invade": 34609, + "invaded": 32596, + "invaders": 35188, + "invading": 40101, + "invali": 31592, + "invalid": 46998, + "invaluable": 33976, + "invasi": 38100, + "invasion": 13378, + "invasive": 19554, + "inve": 2024, + "inven": 26233, + "invent": 11665, + "invent": 23558, + "invented": 14100, + "invention": 23607, + "inventions": 44914, + "inventor": 22836, + "inventory": 19444, + "inver": 12061, + "inverness": 33080, + "inverte": 46397, + "inverted": 40709, + "invest": 4180, + "invest": 9716, + "invested": 22536, + "investig": 4626, + "investigate": 15703, + "investigated": 29180, + "investigates": 29621, + "investigating": 13713, + "investigation": 8194, + "investigations": 24020, + "investigative": 30233, + "investigator": 30528, + "investigators": 24121, + "investin": 40195, + "investing": 10554, + "investment": 5605, + "investments": 14675, + "investor": 15490, + "investors": 10486, + "invests": 38378, + "invic": 25253, + "invigor": 48722, + "invin": 30252, + "invincible": 38052, + "invisible": 16093, + "invit": 12454, + "invitation": 15032, + "invitational": 14511, + "invitations": 40120, + "invite": 8109, + "invited": 7731, + "invites": 16034, + "inviting": 14349, + "invo": 29417, + "invol": 4000, + "involve": 26325, + "involved": 5320, + "involvement": 19502, + "involves": 22652, + "involving": 14786, + "inwx": 35674, + "iny": 23257, + "inyour": 47954, + "io": 3167, + "io": 3752, + "ioc": 43018, + "iom": 33000, + "iom": 31135, + "ion": 14871, + "ion": 3668, + "ions": 26289, + "ior": 7354, + "ior": 2498, + "iority": 46016, + "iors": 6427, + "ios": 6614, + "iot": 32694, + "iot": 6627, + "iota": 37294, + "ious": 6994, + "iously": 38233, + "iow": 7439, + "iowa": 38847, + "iowa": 8290, + "ip": 1719, + "ip": 8600, + "ipa": 11199, + "ipad": 39067, + "ipad": 7491, + "ipads": 35281, + "ipc": 41981, + "iphone": 26030, + "iphone": 4314, + "iphones": 37561, + "ipl": 13440, + "ipment": 37824, + "ipo": 40218, + "ipo": 24090, + "ipod": 17889, + "ipp": 31706, + "ips": 26910, + "ipsw": 22221, + "ipswich": 24494, + "iq": 15554, + "iq": 19996, + "iqbal": 33553, + "ir": 582, + "ir": 742, + "ira": 4923, + "ira": 5371, + "irah": 35724, + "iran": 19273, + "iran": 5075, + "irandeal": 46533, + "irani": 37984, + "iranian": 14158, + "iraq": 8543, + "iraqi": 18617, + "irc": 41527, + "ird": 2770, + "ire": 3013, + "ire": 1454, + "ired": 32728, + "ired": 2995, + "ireland": 32806, + "ireland": 4157, + "irene": 21600, + "ires": 12435, + "irez": 21581, + "irgc": 47942, + "iri": 2155, + "iri": 13880, + "irical": 33366, + "irie": 42979, + "irina": 46664, + "iring": 10169, + "iris": 16437, + "irish": 9386, + "irish": 4889, + "irl": 34494, + "irl": 8570, + "irling": 26493, + "irls": 24344, + "irma": 22406, + "irn": 42603, + "iro": 23209, + "iro": 7280, + "iron": 7699, + "iron": 5391, + "ironic": 24518, + "ironically": 36779, + "ironing": 46655, + "ironman": 20330, + "irons": 30032, + "irony": 20681, + "irport": 27769, + "irr": 24641, + "irrational": 47413, + "irregular": 38692, + "irrelevant": 34677, + "irresi": 31200, + "irresistible": 35252, + "irresponsible": 44714, + "irri": 21484, + "irrigation": 23761, + "irrit": 24218, + "irs": 6086, + "irst": 32701, + "iru": 48206, + "irvin": 47053, + "irvine": 24201, + "irving": 19738, + "irwin": 23750, + "iry": 7239, + "is": 595, + "is": 533, + "isa": 11034, + "isa": 6536, + "isaac": 37544, + "isaac": 13659, + "isab": 13357, + "isabel": 27466, + "isabella": 26192, + "isabelle": 31072, + "isable": 46631, + "isai": 15365, + "isaiah": 17952, + "isak": 40619, + "isance": 46893, + "isation": 7194, + "isback": 43811, + "isc": 39316, + "isch": 47888, + "isco": 5736, + "iscoming": 26458, + "isd": 46816, + "isd": 12002, + "ise": 7669, + "ise": 1479, + "ised": 2861, + "iselle": 48491, + "iser": 23080, + "iser": 5626, + "isers": 34879, + "ises": 5153, + "isf": 44036, + "isgreat": 34595, + "ish": 6844, + "ish": 1061, + "isha": 28050, + "ishable": 37949, + "ished": 35341, + "ishere": 46053, + "ishi": 26224, + "ishq": 27996, + "ishqba": 32503, + "ishqbaaaz": 36591, + "isi": 7233, + "isi": 17880, + "isil": 34636, + "isin": 37676, + "ising": 3426, + "isis": 7531, + "isk": 30171, + "isl": 31368, + "isla": 22807, + "islam": 6003, + "islam": 8770, + "islamabad": 19959, + "islamic": 31627, + "islamic": 9552, + "islamist": 38798, + "islamophobia": 43459, + "island": 13408, + "island": 2619, + "islander": 45651, + "islanders": 27804, + "islands": 7145, + "islay": 49279, + "isle": 19082, + "isle": 11849, + "isleof": 24718, + "isles": 21816, + "islife": 26433, + "islington": 34945, + "ism": 47730, + "ism": 1935, + "isma": 43937, + "ismail": 36140, + "isme": 43570, + "ismo": 41926, + "isms": 18700, + "isn": 2923, + "isner": 48246, + "isnow": 43694, + "isnt": 19416, + "iso": 2462, + "iso": 12263, + "isol": 11414, + "isolated": 19044, + "isolation": 26400, + "ison": 12949, + "ison": 4553, + "isons": 33318, + "isoo": 35857, + "isp": 31397, + "isp": 39041, + "isra": 3591, + "israel": 20837, + "israel": 4779, + "israeli": 8994, + "israelis": 45713, + "isreal": 47147, + "isro": 44841, + "iss": 11738, + "iss": 4950, + "issa": 38579, + "issa": 7560, + "issan": 49358, + "issance": 40828, + "issant": 38828, + "isse": 18986, + "ission": 37946, + "issu": 2049, + "issue": 3202, + "issued": 9246, + "issues": 4082, + "issuing": 37226, + "ist": 9751, + "ist": 2304, + "istanbul": 12258, + "istandwith": 33820, + "iste": 32563, + "ister": 14555, + "isthe": 46748, + "istic": 29556, + "ists": 8426, + "isu": 17030, + "isu": 23328, + "it": 529, + "it": 585, + "ita": 36920, + "ita": 2864, + "itable": 8915, + "ital": 2306, + "ital": 1660, + "itali": 11644, + "italia": 11025, + "italian": 20264, + "italian": 5175, + "italians": 44744, + "italk": 32894, + "italy": 4052, + "itan": 18383, + "itans": 40711, + "itar": 47161, + "itarian": 11599, + "itary": 17604, + "itas": 31634, + "itas": 13436, + "itate": 42457, + "itated": 36744, + "itation": 5070, + "itative": 22892, + "itc": 36449, + "itch": 2387, + "itch": 8147, + "itchen": 32664, + "itchy": 41980, + "ite": 2732, + "ite": 802, + "iteam": 37828, + "itec": 3099, + "itec": 43936, + "itech": 44215, + "itech": 23040, + "ited": 8603, + "ited": 1108, + "itel": 44638, + "itely": 4605, + "item": 8532, + "items": 6207, + "iter": 7938, + "iter": 19773, + "iteracy": 39634, + "iterate": 43106, + "iteration": 38790, + "ites": 2454, + "itez": 42131, + "itf": 35436, + "itfc": 36519, + "ith": 6133, + "ith": 1757, + "ithaca": 46257, + "iti": 760, + "iti": 6165, + "itia": 22634, + "itian": 23365, + "itic": 11950, + "itical": 48767, + "itics": 33967, + "ities": 41423, + "ities": 1480, + "itim": 15676, + "itiner": 32803, + "itinerary": 41564, + "iting": 1257, + "ition": 25263, + "ition": 1104, + "itions": 5540, + "itious": 13329, + "itis": 33539, + "itis": 8388, + "itive": 3067, + "itly": 42240, + "ito": 22167, + "ito": 4661, + "iton": 21119, + "itor": 47267, + "itor": 4584, + "itors": 22005, + "itos": 24560, + "its": 7140, + "its": 902, + "itsa": 45032, + "itself": 7290, + "itsme": 41125, + "itss": 47040, + "itt": 1031, + "itt": 11228, + "itta": 21233, + "itte": 31962, + "itted": 24429, + "itten": 30014, + "itten": 4343, + "itter": 11456, + "itters": 13082, + "itti": 28629, + "ittin": 25646, + "itting": 3147, + "ittle": 24208, + "ittle": 21366, + "ittles": 38989, + "itton": 25707, + "itty": 35096, + "itu": 1668, + "itu": 32128, + "itude": 43382, + "itude": 5012, + "itudes": 20459, + "itunes": 7007, + "itup": 35838, + "iture": 25547, + "itus": 24364, + "itutes": 32883, + "itv": 20159, + "itv": 12805, + "ity": 2480, + "ity": 696, + "itya": 32055, + "itz": 14544, + "itz": 7807, + "iu": 14292, + "iu": 15575, + "ium": 10762, + "ius": 6740, + "iv": 6775, + "iv": 9315, + "iva": 42463, + "ivan": 15544, + "ivan": 15689, + "ivanka": 37914, + "ive": 26885, + "ive": 8653, + "ived": 15654, + "iver": 36849, + "iver": 44254, + "ives": 27333, + "ivf": 39159, + "iving": 45136, + "ivory": 16776, + "ivote": 45835, + "ivy": 36939, + "ivy": 16045, + "iw": 13058, + "iw": 46604, + "iwant": 42747, + "iwd": 16815, + "iwm": 44237, + "ix": 13272, + "ix": 8756, + "iy": 13704, + "iya": 18595, + "iyaki": 48395, + "iz": 2845, + "iz": 8407, + "iza": 37704, + "ization": 10847, + "ize": 10885, + "ized": 7690, + "izen": 34776, + "izer": 23895, + "izes": 45434, + "izing": 17354, + "izo": 46910, + "izz": 31779, + "izz": 46128, + "izzy": 28861, + "j": 73, + "j": 329, + "ja": 1586, + "ja": 2641, + "jaan": 25052, + "jab": 8059, + "jab": 9439, + "jac": 2293, + "jac": 30198, + "jace": 43286, + "jack": 2679, + "jack": 3267, + "jacked": 27923, + "jacket": 6164, + "jackets": 14745, + "jacki": 47418, + "jackie": 28023, + "jackie": 11716, + "jacking": 40929, + "jackman": 35723, + "jackpot": 23926, + "jacks": 19649, + "jackson": 12321, + "jackson": 4363, + "jacksonville": 19263, + "jaco": 6840, + "jacob": 14385, + "jacob": 9222, + "jacobs": 17482, + "jacobson": 46826, + "jacqu": 14495, + "jacqueline": 22843, + "jacques": 17799, + "jad": 12976, + "jad": 38691, + "jada": 37416, + "jade": 25123, + "jade": 14513, + "jaden": 37174, + "jadine": 37445, + "jae": 16869, + "jae": 15765, + "jaejoong": 43610, + "jaf": 19362, + "jag": 7984, + "jag": 36236, + "jagan": 48530, + "jagger": 30835, + "jags": 31086, + "jagu": 10096, + "jaguar": 44777, + "jaguar": 14757, + "jaguars": 21854, + "jah": 20067, + "jah": 11084, + "jahan": 44404, + "jahan": 47827, + "jai": 10542, + "jai": 13819, + "jail": 18574, + "jail": 9332, + "jailbreak": 45990, + "jailed": 19456, + "jails": 47833, + "jaime": 24716, + "jain": 21999, + "jaipur": 23593, + "jais": 48607, + "jait": 28910, + "jaitley": 32776, + "jak": 9225, + "jak": 30589, + "jakarta": 15471, + "jake": 13140, + "jake": 7419, + "jakob": 47358, + "jal": 8380, + "jal": 26773, + "jalan": 27270, + "jalap": 49081, + "jalape": 34263, + "jalapeño": 43017, + "jalen": 33548, + "jam": 1434, + "jam": 5201, + "jama": 8977, + "jama": 35366, + "jamaica": 13019, + "jamaican": 25144, + "jamal": 26108, + "jambo": 35599, + "jamboree": 38506, + "jame": 12341, + "james": 6963, + "james": 2392, + "jamesbond": 44704, + "jamesc": 47004, + "jameson": 31731, + "jami": 15092, + "jamie": 16454, + "jamie": 8078, + "jamiedor": 34310, + "jamiedornan": 34896, + "jammed": 35590, + "jammin": 35223, + "jamming": 25862, + "jammu": 25926, + "jams": 20243, + "jan": 1891, + "jan": 3334, + "jana": 18182, + "jane": 12389, + "jane": 6736, + "janeiro": 31740, + "janet": 29665, + "janet": 15872, + "jang": 41526, + "jang": 22074, + "jani": 22606, + "janice": 36048, + "janine": 46896, + "janis": 44233, + "jann": 35377, + "jans": 22578, + "jansen": 45354, + "janu": 3623, + "january": 3697, + "jap": 2299, + "jap": 49062, + "japan": 4502, + "japan": 3400, + "japanese": 27211, + "japanese": 4925, + "japs": 42121, + "jar": 5120, + "jar": 10837, + "jard": 25778, + "jardin": 37371, + "jare": 17654, + "jared": 35597, + "jared": 12571, + "jaredle": 36739, + "jaredleto": 37106, + "jaro": 35505, + "jarpad": 44497, + "jarre": 23385, + "jarrett": 30531, + "jars": 27583, + "jarvis": 29286, + "jas": 4492, + "jas": 17559, + "jasmin": 42989, + "jasmin": 47700, + "jasmine": 17056, + "jason": 10009, + "jason": 5395, + "jasper": 19827, + "jat": 26106, + "jau": 26932, + "jauregui": 48175, + "jav": 6234, + "java": 12918, + "javascri": 16289, + "javascript": 16423, + "jave": 46218, + "javed": 42268, + "javelin": 41701, + "javi": 47627, + "javier": 23307, + "jaw": 14804, + "jaw": 17307, + "jawa": 44790, + "jaws": 25491, + "jax": 22348, + "jax": 12390, + "jay": 3427, + "jay": 4155, + "jaya": 21960, + "jayanti": 37732, + "jaye": 45703, + "jayne": 35228, + "jays": 12393, + "jaz": 3465, + "jaz": 32874, + "jazeera": 38260, + "jazz": 11488, + "jazz": 4528, + "jazzfest": 36683, + "jazzy": 28191, + "jb": 21915, + "jb": 13637, + "jc": 14991, + "jc": 11517, + "jd": 18289, + "jd": 14125, + "jdm": 42013, + "je": 1013, + "je": 8776, + "jeal": 9964, + "jealous": 11093, + "jealousy": 37654, + "jean": 13943, + "jean": 6473, + "jeanette": 48167, + "jeanne": 29201, + "jeans": 10157, + "jeb": 35101, + "jec": 1347, + "ject": 6070, + "jed": 12166, + "jed": 38748, + "jeddah": 40982, + "jedi": 16681, + "jee": 29250, + "jee": 14870, + "jeep": 16593, + "jeep": 11286, + "jeeplife": 43100, + "jeet": 45542, + "jeet": 30944, + "jef": 10276, + "jeff": 6245, + "jeff": 5550, + "jefferson": 44711, + "jefferson": 13976, + "jeffery": 41470, + "jeffree": 45994, + "jeffrey": 32886, + "jeffrey": 16027, + "jeho": 42437, + "jeky": 43893, + "jekyll": 49405, + "jel": 9794, + "jelena": 48218, + "jelly": 19110, + "jelly": 13762, + "jellyfish": 30988, + "jem": 46326, + "jem": 37530, + "jen": 2554, + "jen": 12997, + "jenkins": 16162, + "jenn": 33921, + "jenn": 29869, + "jenna": 17125, + "jenner": 14260, + "jenni": 6774, + "jennie": 28875, + "jennifer": 19786, + "jennifer": 8613, + "jennings": 21564, + "jenny": 20165, + "jenny": 13414, + "jens": 40806, + "jensen": 35558, + "jensen": 19004, + "jensenackles": 41011, + "jeon": 45200, + "jeon": 43337, + "jeong": 47146, + "jeong": 39264, + "jeopar": 22988, + "jeopardy": 29613, + "jer": 2310, + "jer": 35307, + "jere": 5614, + "jeremi": 22362, + "jeremiah": 27301, + "jeremy": 14656, + "jeremy": 8127, + "jeremycorbyn": 37484, + "jeric": 25084, + "jericho": 28892, + "jerk": 23917, + "jerky": 40079, + "jermaine": 40722, + "jerome": 19876, + "jerry": 18163, + "jerry": 9164, + "jersey": 21921, + "jersey": 4471, + "jerseys": 15518, + "jerus": 12257, + "jerusalem": 12557, + "jes": 7686, + "jes": 35826, + "jess": 5313, + "jess": 13758, + "jesse": 23112, + "jesse": 11770, + "jessi": 24373, + "jessic": 14881, + "jessica": 45421, + "jessica": 8178, + "jessie": 19424, + "jester": 44225, + "jesu": 19777, + "jesuit": 33234, + "jesus": 4070, + "jet": 11515, + "jet": 6565, + "jetblue": 45021, + "jeter": 38450, + "jets": 38584, + "jets": 10025, + "jett": 44541, + "jetty": 46382, + "jew": 27450, + "jewel": 4880, + "jewel": 17591, + "jewell": 9777, + "jewellers": 46265, + "jewellery": 11192, + "jewelry": 28018, + "jewelry": 6039, + "jewels": 20205, + "jewish": 29594, + "jewish": 9104, + "jews": 14200, + "jf": 31130, + "jf": 33718, + "jfc": 43652, + "jfk": 18486, + "jg": 41986, + "jg": 35138, + "jh": 24858, + "jh": 21485, + "jha": 47012, + "jha": 38092, + "jhal": 45695, + "jhar": 31546, + "jharkhand": 39001, + "jhb": 34631, + "ji": 3252, + "ji": 2697, + "jia": 32907, + "jian": 33427, + "jiang": 43309, + "jiang": 25762, + "jic": 48350, + "jic": 40215, + "jid": 24403, + "jie": 40005, + "jig": 15136, + "jig": 47430, + "jigsaw": 32987, + "jiha": 23194, + "jihad": 29637, + "jihoon": 44765, + "jil": 36225, + "jill": 24136, + "jill": 15254, + "jillian": 37820, + "jim": 3190, + "jim": 4550, + "jima": 20679, + "jimcantore": 43950, + "jimenez": 35947, + "jimi": 30565, + "jimin": 16286, + "jimmie": 45679, + "jimmy": 12215, + "jimmy": 6817, + "jimmyfallon": 45265, + "jin": 7927, + "jin": 8485, + "jind": 40609, + "jing": 34933, + "jing": 28607, + "jingle": 28699, + "jinnah": 43141, + "jinping": 39308, + "jinx": 42977, + "jinyoung": 38051, + "jio": 40501, + "jis": 25988, + "jis": 23515, + "jisoo": 43070, + "jit": 11947, + "jit": 20308, + "jitsu": 24530, + "jiu": 43351, + "jiu": 44123, + "jj": 12502, + "jj": 12790, + "jk": 20189, + "jk": 9702, + "jkt": 21494, + "jl": 25027, + "jl": 22911, + "jlo": 31017, + "jm": 24044, + "jm": 18657, + "jn": 24576, + "jn": 21717, + "jnr": 37145, + "jnu": 47142, + "jo": 683, + "jo": 3804, + "joachim": 48979, + "joan": 28064, + "joan": 12710, + "joann": 35484, + "joanna": 25357, + "joanne": 43736, + "joanne": 25092, + "joao": 45666, + "joaqu": 25140, + "joaquin": 30745, + "job": 13114, + "job": 2075, + "jobs": 3735, + "jobsearch": 45459, + "joburg": 39343, + "jocel": 36879, + "jocelyn": 47259, + "jock": 34485, + "jockey": 20126, + "jodh": 48689, + "jodi": 36812, + "jodi": 26888, + "jodie": 33100, + "jody": 32959, + "joe": 9309, + "joe": 3305, + "joel": 19819, + "joel": 11429, + "joes": 34756, + "joey": 16281, + "joey": 10455, + "jog": 37967, + "jog": 31691, + "jogging": 37922, + "joh": 1201, + "johan": 17416, + "johan": 27789, + "johann": 31180, + "johanna": 41494, + "johannes": 37779, + "johannesburg": 28377, + "johansson": 41512, + "johar": 34871, + "john": 2004, + "john": 1742, + "johncena": 46820, + "johnnie": 47947, + "johnny": 14464, + "johnny": 6904, + "johns": 14515, + "johnson": 26036, + "johnson": 4010, + "johnston": 19791, + "johnstone": 40766, + "johor": 34750, + "join": 14737, + "join": 1384, + "joined": 4954, + "joining": 5118, + "joins": 5681, + "joint": 6640, + "jointhe": 30422, + "jointly": 37471, + "joints": 27204, + "jojo": 41484, + "jojo": 22075, + "joke": 7198, + "joker": 18200, + "jokers": 44101, + "jokes": 11336, + "joking": 26112, + "joko": 44975, + "jol": 9174, + "jol": 36470, + "jolie": 31633, + "jolla": 46109, + "jolly": 21516, + "jom": 32152, + "jon": 3026, + "jon": 6139, + "jona": 6629, + "jonah": 47934, + "jonah": 27556, + "jonas": 42373, + "jonas": 13650, + "jonathan": 19026, + "jonathan": 7762, + "jone": 33934, + "jones": 19091, + "jones": 3538, + "jong": 20214, + "jong": 14726, + "jonghyun": 29023, + "jongin": 36957, + "joni": 43177, + "jonny": 28454, + "jonny": 21895, + "joo": 25807, + "joo": 27680, + "joom": 47543, + "joon": 18547, + "joong": 26544, + "jop": 30486, + "joplin": 42688, + "jor": 2482, + "jor": 31595, + "jordan": 14644, + "jordan": 4388, + "jordani": 46898, + "jordi": 44795, + "jorge": 48761, + "jorge": 18225, + "jos": 20560, + "jos": 19661, + "jose": 4647, + "jose": 7075, + "josef": 36584, + "josel": 47800, + "joseph": 14163, + "joseph": 6478, + "josephine": 34866, + "josh": 9998, + "josh": 5679, + "joshi": 24786, + "joshu": 9112, + "joshua": 11852, + "josi": 33583, + "josie": 33167, + "joss": 42834, + "josé": 27922, + "jou": 19921, + "jou": 32029, + "jour": 2078, + "jour": 17142, + "journ": 4563, + "journal": 6626, + "journalism": 10123, + "journalist": 9914, + "journalists": 12249, + "journals": 24391, + "journe": 48833, + "journey": 32156, + "journey": 3749, + "journeys": 23329, + "journo": 37034, + "journos": 46437, + "jovi": 33866, + "joy": 6308, + "joy": 4273, + "joyce": 43753, + "joyce": 15275, + "joye": 34052, + "joyeux": 41876, + "joyful": 24139, + "joyous": 32245, + "joyride": 46949, + "joys": 22996, + "jp": 18249, + "jp": 10557, + "jpg": 36950, + "jpn": 36212, + "jr": 13973, + "jr": 3605, + "js": 46243, + "js": 8006, + "jst": 26523, + "jt": 39480, + "jt": 18119, + "ju": 669, + "ju": 9970, + "jual": 38720, + "juan": 17148, + "juan": 9274, + "juana": 9081, + "jubi": 15485, + "jubil": 47743, + "jubilee": 16907, + "juco": 31570, + "jud": 8363, + "juda": 32478, + "judah": 41066, + "judaism": 42217, + "judas": 39532, + "judd": 29770, + "judg": 20012, + "judge": 16824, + "judge": 5656, + "judged": 33453, + "judgement": 25246, + "judges": 12575, + "judging": 16570, + "judgment": 24191, + "judi": 42546, + "judice": 28032, + "judicial": 19579, + "judiciary": 24545, + "judith": 24047, + "judo": 27011, + "judy": 34663, + "judy": 16510, + "jug": 27619, + "jugg": 38628, + "juic": 38761, + "juice": 37954, + "juice": 6916, + "juices": 36757, + "juicy": 17623, + "juju": 43020, + "juke": 32519, + "jukebox": 36411, + "jul": 34662, + "jul": 15975, + "jule": 40819, + "jules": 21996, + "juli": 3614, + "juli": 49160, + "julia": 10207, + "julian": 25459, + "julian": 12643, + "juliana": 46059, + "julie": 22534, + "julie": 10505, + "julien": 32595, + "juliet": 20641, + "juliette": 44804, + "julio": 24888, + "julius": 20870, + "july": 2272, + "jum": 20791, + "jumbo": 24678, + "jume": 45989, + "jump": 5519, + "jump": 6423, + "jumped": 16901, + "jumper": 16558, + "jumpers": 36485, + "jumping": 11476, + "jumpman": 48803, + "jumps": 18911, + "jumpsuit": 31044, + "jun": 1637, + "jun": 7719, + "junction": 11320, + "june": 23188, + "june": 2345, + "jung": 13086, + "jung": 13031, + "jungkook": 20040, + "jungle": 42421, + "jungle": 10865, + "juni": 4029, + "junior": 21167, + "junior": 5027, + "juniors": 16811, + "juniper": 33829, + "junk": 16000, + "junkie": 27613, + "junkies": 41207, + "juno": 28845, + "junto": 34282, + "jupit": 15270, + "jupiter": 16212, + "jur": 15896, + "jura": 14715, + "jurassic": 28844, + "jurassic": 21255, + "jurgen": 39263, + "juris": 37010, + "jurisdic": 37714, + "jury": 12931, + "jus": 14999, + "just": 1770, + "just": 761, + "justi": 14700, + "justic": 30399, + "justice": 16904, + "justice": 3604, + "justicefor": 25812, + "justiceleague": 41929, + "justices": 44356, + "justified": 34546, + "justify": 28192, + "justin": 7537, + "justin": 4394, + "justinbieber": 12501, + "justine": 34418, + "justintrudeau": 32184, + "justsaying": 42922, + "juve": 47717, + "juve": 23092, + "juven": 12944, + "juvenile": 19333, + "juvent": 13908, + "juventus": 47378, + "juventus": 16208, + "jux": 33552, + "juxta": 34964, + "jv": 37932, + "jv": 11805, + "jw": 30221, + "jw": 24215, + "jy": 20979, + "jyo": 27378, + "jyoti": 48696, + "jä": 45381, + "k": 74, + "k": 330, + "ka": 1595, + "ka": 1525, + "kaa": 34496, + "kab": 6554, + "kab": 45134, + "kabaddi": 41749, + "kabir": 38619, + "kabo": 47974, + "kabul": 26160, + "kac": 21693, + "kach": 14341, + "kad": 10901, + "kade": 41130, + "kaduna": 38053, + "kae": 22542, + "kaeper": 30070, + "kaepernick": 30713, + "kaf": 19870, + "kag": 13666, + "kag": 31003, + "kah": 16068, + "kah": 15463, + "kahn": 35397, + "kai": 12752, + "kai": 9601, + "kaido": 40255, + "kail": 23623, + "kaine": 39028, + "kair": 33027, + "kaiser": 43685, + "kaiser": 29960, + "kait": 19326, + "kaitlyn": 34948, + "kaj": 44788, + "kaj": 40381, + "kak": 10401, + "kak": 40128, + "kaka": 47689, + "kaku": 30900, + "kal": 4187, + "kal": 18712, + "kala": 45453, + "kala": 33105, + "kalam": 40142, + "kalamaz": 42328, + "kalamazoo": 46264, + "kalb": 34483, + "kale": 17162, + "kale": 16625, + "kaleido": 41144, + "kali": 17844, + "kali": 26964, + "kalin": 42776, + "kalyan": 23825, + "kam": 4104, + "kam": 26011, + "kamal": 31371, + "kamal": 28619, + "kamala": 45003, + "kame": 45235, + "kamen": 40738, + "kami": 28707, + "kamloops": 36602, + "kamp": 35179, + "kamp": 29522, + "kampala": 37134, + "kan": 2532, + "kan": 8101, + "kana": 35178, + "kand": 17478, + "kane": 32218, + "kane": 9765, + "kang": 12226, + "kang": 20789, + "kangar": 20622, + "kangaroo": 25513, + "kani": 40907, + "kani": 41948, + "kann": 18533, + "kannada": 30053, + "kano": 28201, + "kans": 34012, + "kansas": 25507, + "kansas": 6539, + "kansascity": 46134, + "kant": 39923, + "kant": 47132, + "kanth": 24427, + "kanu": 44565, + "kany": 13590, + "kanye": 29680, + "kanye": 14965, + "kanyewest": 31943, + "kap": 6804, + "kap": 45279, + "kapam": 48561, + "kapil": 32337, + "kapil": 42709, + "kapilshar": 48978, + "kaplan": 37401, + "kapoor": 9117, + "kapp": 36717, + "kappa": 20239, + "kapur": 42371, + "kar": 1813, + "kar": 5933, + "kara": 12552, + "karab": 40916, + "karachi": 13671, + "karak": 40372, + "karan": 20077, + "karan": 20931, + "karanjohar": 47621, + "karao": 16262, + "karaoke": 16640, + "karate": 21211, + "kardashi": 13619, + "kardashian": 14578, + "kare": 14310, + "kare": 38354, + "kareem": 38885, + "kareena": 41569, + "karen": 17719, + "karen": 10349, + "kari": 15339, + "kari": 15161, + "karim": 33477, + "karin": 43917, + "karina": 40250, + "karl": 20967, + "karl": 13134, + "karla": 42309, + "karma": 17658, + "karnat": 13994, + "karnataka": 15515, + "karo": 45305, + "kart": 47841, + "kart": 21310, + "karthik": 41397, + "karti": 23053, + "kartikeyan": 32584, + "karting": 41655, + "kas": 6119, + "kas": 14372, + "kasa": 46111, + "kash": 6954, + "kash": 21371, + "kashi": 47945, + "kashmir": 20251, + "kashmir": 10783, + "kashmiri": 35331, + "kasi": 45870, + "kasi": 32819, + "kasich": 39666, + "kat": 2844, + "kat": 9341, + "kata": 14558, + "kate": 11620, + "kate": 6699, + "katelyn": 45963, + "kath": 7386, + "kath": 19745, + "katharine": 41473, + "katherine": 17687, + "kathle": 18721, + "kathleen": 21709, + "kathmandu": 34456, + "kathniel": 36159, + "kathr": 14905, + "kathryn": 33142, + "kathryn": 19999, + "kathy": 34775, + "kathy": 18795, + "kati": 6515, + "kati": 29928, + "katic": 48058, + "katie": 24117, + "katie": 9076, + "katniss": 47916, + "kato": 27573, + "katrin": 31282, + "katrina": 21397, + "katrinakaif": 45845, + "kats": 44213, + "katsu": 49296, + "katsu": 43712, + "katy": 17609, + "katy": 14435, + "katyperry": 28309, + "katz": 30790, + "kau": 9299, + "kau": 36895, + "kauai": 44050, + "kaufman": 37188, + "kaur": 30518, + "kav": 10228, + "kavan": 18576, + "kavanaugh": 20252, + "kaw": 10842, + "kaw": 42719, + "kawa": 33244, + "kawaii": 26891, + "kawasaki": 28227, + "kawhi": 41220, + "kay": 4673, + "kay": 9862, + "kaya": 22752, + "kayak": 27043, + "kayaking": 28977, + "kaye": 33003, + "kayla": 17139, + "kaylee": 47215, + "kayo": 37021, + "kaz": 8812, + "kaz": 39622, + "kazakh": 25451, + "kazakhstan": 26720, + "kazan": 47641, + "kb": 27381, + "kb": 19960, + "kbs": 27418, + "kc": 10869, + "kc": 8638, + "kca": 14347, + "kcon": 39970, + "kcr": 46181, + "kd": 21826, + "kd": 15597, + "kday": 31074, + "kdrama": 48628, + "ke": 643, + "ke": 618, + "kea": 47926, + "kean": 43288, + "keane": 28635, + "keanu": 40608, + "kear": 21562, + "kearney": 36435, + "keating": 40045, + "keaton": 29975, + "kebab": 36497, + "ked": 11730, + "ked": 1243, + "kee": 9724, + "kee": 6760, + "keef": 42323, + "keefe": 46965, + "keegan": 31122, + "keel": 48376, + "keen": 17714, + "keen": 13218, + "keenan": 36276, + "keep": 2924, + "keep": 1726, + "keeper": 7650, + "keepers": 16130, + "keepin": 41712, + "keeping": 38371, + "keeping": 4873, + "keepit": 28044, + "keeps": 6333, + "keer": 27412, + "keerth": 47500, + "keerthyofficial": 48185, + "kees": 10791, + "keg": 32785, + "keh": 41272, + "keh": 36983, + "kei": 18735, + "kei": 24835, + "keith": 18762, + "keith": 8252, + "kej": 15674, + "kejri": 16617, + "kejriwal": 17334, + "keke": 39195, + "kel": 2825, + "kel": 7553, + "kele": 41765, + "kell": 16082, + "kell": 40103, + "keller": 21407, + "kelley": 23776, + "kelli": 45852, + "kelli": 46190, + "kellie": 49224, + "kellogg": 44218, + "kelly": 13417, + "kelly": 5220, + "kelown": 31708, + "kelowna": 32963, + "kelsey": 42295, + "kelsey": 23018, + "kelvin": 32859, + "kem": 31013, + "kem": 17349, + "kemp": 18302, + "kemp": 25325, + "ken": 1838, + "ken": 1702, + "kend": 7497, + "kendal": 44836, + "kendall": 34607, + "kendall": 16238, + "kendra": 36074, + "kendrick": 41787, + "kendrick": 21953, + "kendricklamar": 47020, + "kenne": 6209, + "kennedy": 38631, + "kennedy": 9004, + "kennel": 39595, + "kenneth": 46900, + "kenneth": 17839, + "kenney": 41373, + "kenny": 20185, + "kenny": 9595, + "kens": 29765, + "kensing": 21505, + "kensington": 24988, + "kent": 13875, + "kent": 8214, + "kentu": 9045, + "kentucky": 32230, + "kentucky": 10014, + "keny": 17374, + "kenya": 6181, + "kenyan": 22624, + "kenyans": 36263, + "kenyatta": 31012, + "kenzie": 38087, + "keo": 43062, + "kept": 7737, + "ker": 2352, + "ker": 1485, + "keral": 35122, + "kerala": 11881, + "kered": 26690, + "kerel": 32232, + "keri": 43447, + "kermit": 40908, + "kern": 40150, + "kernel": 40684, + "kerr": 20491, + "kerri": 41849, + "kerry": 24795, + "kerry": 13097, + "kers": 30347, + "kers": 2880, + "kershaw": 40785, + "kerson": 42810, + "kerswednesday": 48152, + "kert": 47279, + "kes": 38398, + "kes": 1115, + "kesh": 19751, + "kesha": 36526, + "kest": 15080, + "ket": 2715, + "ket": 1236, + "ketball": 38240, + "ketch": 22590, + "ketch": 35371, + "ketchup": 26724, + "kete": 25404, + "keted": 41396, + "keting": 15951, + "keto": 27485, + "keto": 28754, + "kets": 1632, + "kett": 23124, + "kett": 10312, + "kettering": 43779, + "kettle": 41992, + "kettle": 24303, + "kev": 22758, + "kev": 29419, + "kevin": 9419, + "kevin": 4685, + "kew": 38014, + "kew": 31409, + "kex": 30251, + "key": 2891, + "key": 1458, + "keyan": 27617, + "keyboard": 13017, + "keyboards": 49237, + "keychain": 31050, + "keye": 40516, + "keye": 20635, + "keyes": 18336, + "keynes": 32462, + "keynote": 7556, + "keys": 48912, + "keys": 6355, + "keystone": 30688, + "keyword": 42284, + "keywords": 48122, + "kf": 33308, + "kf": 42119, + "kfc": 22032, + "kg": 36772, + "kg": 7817, + "kgs": 46629, + "kh": 2166, + "kh": 7452, + "kha": 7333, + "kha": 18929, + "khair": 43742, + "khaki": 41646, + "khal": 13070, + "khaled": 29343, + "khali": 11324, + "khalid": 27166, + "khalifa": 21389, + "khalil": 36229, + "kham": 24892, + "khan": 13318, + "khan": 3873, + "khand": 43384, + "khand": 31110, + "khanna": 29931, + "khar": 18340, + "khar": 28578, + "khart": 37458, + "khat": 43290, + "khe": 26360, + "kher": 43843, + "khi": 39062, + "khi": 42925, + "khil": 34101, + "khloe": 45312, + "kho": 14022, + "kho": 28774, + "khou": 30656, + "khs": 21239, + "khtar": 45593, + "khu": 14041, + "khur": 32083, + "khy": 40917, + "khz": 45604, + "ki": 848, + "ki": 2608, + "kia": 8712, + "kian": 43961, + "kian": 25708, + "kians": 44010, + "kib": 43108, + "kiba": 37207, + "kic": 24003, + "kic": 27633, + "kicchasu": 44665, + "kicchasudeep": 45560, + "kick": 4102, + "kick": 4289, + "kickass": 39299, + "kickboxing": 36041, + "kicked": 12479, + "kicker": 26338, + "kickin": 34597, + "kicking": 7802, + "kickoff": 10245, + "kicks": 6989, + "kickstart": 40780, + "kickstarter": 13228, + "kid": 3948, + "kid": 3551, + "kidd": 24082, + "kidding": 14535, + "kiddo": 36360, + "kiddos": 29205, + "kidlit": 39064, + "kidlit": 33515, + "kidlitart": 41600, + "kidman": 44931, + "kidnap": 45100, + "kidnapp": 16183, + "kidnapped": 24737, + "kidnapping": 32361, + "kidney": 37835, + "kidney": 14610, + "kids": 15561, + "kids": 1911, + "kidz": 41938, + "kie": 8544, + "kie": 3094, + "kiefer": 48026, + "kiel": 40940, + "kiel": 25509, + "kien": 28782, + "kier": 20403, + "kier": 35575, + "kieran": 29231, + "kies": 36601, + "kies": 4993, + "kiest": 29755, + "kiev": 24585, + "kiewicz": 47574, + "kigali": 40278, + "kii": 39340, + "kik": 36176, + "kiki": 23962, + "kiko": 40861, + "kil": 4912, + "kil": 39337, + "kildare": 45541, + "kili": 24386, + "kilig": 49172, + "kilimanjaro": 43470, + "kilkenny": 33805, + "kill": 6163, + "kill": 4367, + "killa": 41355, + "killarney": 48813, + "killed": 3733, + "killer": 28230, + "killer": 6613, + "killers": 17614, + "killin": 25903, + "killing": 37977, + "killing": 5923, + "killings": 24918, + "kills": 9795, + "kiln": 44150, + "kilo": 39281, + "kilom": 26285, + "kilometers": 39192, + "kilometres": 43278, + "kilt": 49319, + "kim": 4639, + "kim": 4606, + "kimber": 16796, + "kimberley": 39859, + "kimberly": 27465, + "kimchi": 41027, + "kimi": 31536, + "kimkardashian": 35400, + "kimmel": 27820, + "kimono": 40024, + "kin": 1442, + "kin": 2667, + "kina": 28518, + "kind": 7204, + "kind": 3044, + "kinda": 6612, + "kinder": 12711, + "kinder": 24159, + "kindergarten": 16749, + "kindle": 24704, + "kindle": 10746, + "kindleunlimited": 32164, + "kindly": 13952, + "kindness": 45112, + "kindness": 10614, + "kinds": 14879, + "kine": 17607, + "kineni": 49080, + "kinetic": 37699, + "king": 2365, + "king": 674, + "kingdom": 21870, + "kingdom": 7364, + "kingdomhearts": 48570, + "kingdoms": 43890, + "kingfisher": 34330, + "kingjames": 33153, + "kingly": 33642, + "kingof": 27878, + "kings": 18590, + "kings": 4232, + "kingsley": 41807, + "kingston": 40736, + "kingston": 15393, + "kini": 41644, + "kinky": 37006, + "kinney": 37233, + "kino": 39000, + "kins": 31060, + "kins": 4386, + "kinson": 12095, + "kio": 28210, + "kio": 39401, + "kiosk": 39146, + "kip": 27636, + "kip": 15986, + "kipp": 43329, + "kir": 3476, + "kir": 32949, + "kira": 33038, + "kiran": 43234, + "kiran": 36603, + "kirby": 17065, + "kiri": 34170, + "kiri": 45826, + "kirk": 10639, + "kirk": 11508, + "kirkland": 43061, + "kiro": 39749, + "kirstel": 46483, + "kirsten": 31813, + "kirsty": 37787, + "kis": 3199, + "kis": 22796, + "kish": 25662, + "kiss": 43757, + "kiss": 5946, + "kissed": 22561, + "kisses": 47876, + "kisses": 11220, + "kissing": 18637, + "kistan": 29580, + "kit": 4566, + "kit": 4274, + "kita": 29961, + "kitch": 3850, + "kitchen": 18131, + "kitchen": 4485, + "kitchener": 34428, + "kitchens": 28301, + "kite": 47777, + "kite": 19867, + "kites": 45829, + "kits": 13730, + "kitt": 10840, + "kitten": 13063, + "kittens": 17216, + "kitties": 36013, + "kitty": 25067, + "kitty": 8417, + "kiwan": 38709, + "kiwanis": 46513, + "kiwi": 22440, + "kiwis": 48108, + "kiya": 41610, + "kj": 27385, + "kj": 28238, + "kja": 41048, + "kjv": 37387, + "kk": 4390, + "kk": 10849, + "kka": 19002, + "kke": 44239, + "kker": 32399, + "kki": 44672, + "kkk": 20073, + "kkkk": 15834, + "kkkk": 47160, + "kkkkkkkk": 31042, + "kko": 43965, + "kkr": 40855, + "kl": 8498, + "kl": 14134, + "kla": 11249, + "klan": 46935, + "klar": 41374, + "klaus": 31788, + "kle": 7612, + "kle": 7432, + "klein": 33475, + "klein": 17579, + "kley": 18594, + "kli": 31640, + "klin": 44809, + "klin": 41647, + "kline": 47580, + "kling": 40270, + "klm": 38859, + "klo": 15296, + "klopp": 26446, + "kltu": 25978, + "klu": 21852, + "kly": 45090, + "km": 29954, + "km": 4590, + "kman": 33312, + "kms": 24996, + "kn": 4825, + "kn": 23693, + "knapp": 33945, + "kne": 6358, + "knee": 9897, + "knees": 19115, + "kner": 31578, + "knew": 5009, + "kni": 6312, + "knick": 33286, + "knicks": 17657, + "knife": 44176, + "knife": 8960, + "knigh": 43099, + "knight": 17949, + "knight": 7355, + "knights": 10385, + "knit": 18745, + "knit": 14313, + "knitted": 28151, + "knitting": 18863, + "knives": 20910, + "kno": 1482, + "kno": 25362, + "knob": 29736, + "knobs": 47504, + "knock": 14195, + "knock": 11583, + "knocked": 15325, + "knocking": 20380, + "knockout": 22602, + "knocks": 24296, + "knoll": 43882, + "knot": 18412, + "knots": 32428, + "know": 4179, + "know": 1038, + "knowing": 9267, + "knowledge": 27864, + "knowledge": 5510, + "knowledgeable": 43391, + "knowles": 32631, + "known": 3102, + "knows": 4309, + "knowyour": 30773, + "knox": 18630, + "knox": 21833, + "knoxville": 23232, + "knu": 14812, + "knuck": 21333, + "knuckle": 42023, + "knuckles": 40127, + "knw": 40803, + "ko": 1313, + "ko": 2448, + "koala": 36654, + "kobe": 42644, + "kobe": 14470, + "kobo": 42390, + "koch": 25331, + "kochi": 36710, + "kodak": 30425, + "kodi": 46611, + "kof": 17528, + "koff": 47303, + "kofi": 40400, + "koh": 13379, + "koh": 31216, + "kohl": 48479, + "kohli": 17549, + "koi": 28150, + "kojima": 46419, + "kok": 32045, + "kok": 11225, + "koko": 42426, + "koko": 40003, + "kol": 7142, + "kol": 31023, + "kolkata": 18011, + "kom": 6686, + "kom": 24181, + "kombat": 29670, + "kombucha": 48615, + "komo": 31820, + "kon": 5743, + "kon": 29519, + "kona": 30203, + "kong": 31784, + "kong": 6506, + "konstant": 46583, + "koo": 12225, + "koo": 40472, + "kook": 16003, + "kool": 36755, + "kool": 26444, + "kop": 16623, + "kop": 38999, + "kor": 6428, + "kor": 24175, + "kore": 3919, + "korea": 5915, + "korean": 31949, + "korean": 8034, + "kori": 42842, + "korn": 45412, + "korn": 31492, + "kors": 34535, + "kos": 47438, + "kos": 22951, + "kosh": 45233, + "kosher": 36502, + "koso": 23892, + "kosovo": 28343, + "kot": 23323, + "kot": 20701, + "kota": 21735, + "koto": 40945, + "koto": 29977, + "kou": 18502, + "kou": 39614, + "kour": 34134, + "kov": 17733, + "kov": 15156, + "kova": 26185, + "koval": 47903, + "kovic": 16886, + "kovich": 44794, + "kovsky": 33384, + "kow": 29764, + "kow": 23919, + "kowski": 17649, + "koz": 29598, + "kp": 16174, + "kp": 16894, + "kpa": 38759, + "kph": 41138, + "kpk": 42094, + "kpmg": 38243, + "kpop": 29534, + "kpop": 15859, + "kprc": 47832, + "kprs": 46253, + "kr": 7309, + "kr": 14107, + "kra": 5762, + "kraft": 28057, + "kraja": 29016, + "kraken": 48408, + "krakow": 40033, + "kram": 19075, + "kramer": 27495, + "kran": 33243, + "kranti": 47969, + "krat": 30470, + "kre": 8362, + "kreme": 43140, + "kremlin": 33979, + "kri": 3679, + "kris": 35251, + "kris": 12261, + "krish": 11487, + "krishna": 15863, + "krishnan": 46535, + "krispy": 49292, + "krist": 16490, + "kristen": 28881, + "kristen": 16644, + "kristi": 26895, + "kristin": 35408, + "kristin": 26785, + "kristina": 33180, + "krit": 36265, + "kro": 16193, + "kroger": 36344, + "kron": 25999, + "kru": 10609, + "kruger": 32948, + "krun": 43084, + "kry": 13995, + "krystal": 36554, + "ks": 10470, + "ks": 662, + "ksa": 25439, + "ksh": 36594, + "kst": 17420, + "kstate": 48590, + "ksu": 43496, + "kswx": 36180, + "kt": 17238, + "kt": 7792, + "ktm": 33989, + "ktn": 42170, + "kton": 37848, + "kts": 48577, + "ktv": 36444, + "ku": 1836, + "ku": 4827, + "kuala": 30336, + "kubball": 48995, + "kuber": 41336, + "kubernetes": 45144, + "kubrick": 37032, + "kuch": 39394, + "kud": 40818, + "kudos": 14481, + "kul": 11325, + "kul": 31514, + "kum": 18086, + "kum": 28148, + "kuma": 43139, + "kuma": 33920, + "kumar": 22329, + "kumar": 7674, + "kumb": 31391, + "kun": 6849, + "kun": 21842, + "kung": 39656, + "kung": 22347, + "kunst": 37881, + "kup": 39023, + "kups": 27240, + "kur": 4862, + "kurdi": 23504, + "kurdish": 21644, + "kurdistan": 24459, + "kurds": 20888, + "kuri": 46375, + "kuro": 28239, + "kuro": 47826, + "kurt": 31903, + "kurt": 14527, + "kus": 27618, + "kus": 27505, + "kush": 22264, + "kush": 24594, + "kushner": 36716, + "kut": 17283, + "kut": 36965, + "kuwait": 19679, + "kuya": 34815, + "kuz": 33253, + "kv": 27594, + "kv": 34249, + "kw": 10072, + "kw": 18339, + "kwa": 32784, + "kwa": 48576, + "kwame": 46681, + "kwan": 37100, + "kwan": 39447, + "kwang": 40260, + "kwe": 26050, + "kwi": 35327, + "kwon": 36369, + "kx": 28190, + "kx": 46442, + "ky": 2018, + "ky": 2383, + "kya": 29142, + "kyc": 37758, + "kyiv": 36422, + "kyle": 15847, + "kyle": 7539, + "kylie": 28282, + "kylie": 17983, + "kyliejenner": 47232, + "kylo": 47704, + "kyo": 13150, + "kyo": 6281, + "kyoto": 23223, + "kyr": 26329, + "kyrgy": 40013, + "kyrgyz": 48346, + "kyrie": 21857, + "kyu": 28296, + "kyu": 25490, + "kyuhyun": 37229, + "kyung": 41058, + "kyungsoo": 30280, + "kywx": 39940, + "kz": 48743, + "kz": 36848, + "kzn": 38264, + "kö": 32437, + "l": 75, + "l": 331, + "la": 572, + "la": 1210, + "laa": 44642, + "lab": 3537, + "lab": 4352, + "labe": 25749, + "label": 12235, + "label": 9093, + "labeled": 32720, + "labeling": 36825, + "labelled": 45188, + "labels": 17413, + "lable": 31879, + "labor": 11201, + "labor": 7878, + "laboratories": 43421, + "laboratory": 17664, + "laborday": 39324, + "labou": 32700, + "labour": 19586, + "labour": 6019, + "labourdoorstep": 37008, + "labout": 35961, + "labra": 37067, + "labrador": 25409, + "labs": 12021, + "laby": 29131, + "labyrin": 31782, + "labyrinth": 35594, + "lac": 4477, + "lac": 16189, + "lace": 30012, + "lace": 5421, + "laced": 36800, + "laces": 23281, + "lacey": 31754, + "lach": 30558, + "lack": 24915, + "lack": 8069, + "lacking": 30080, + "lacks": 34388, + "laco": 45882, + "lacrosse": 12915, + "lacy": 38645, + "lad": 15991, + "lad": 10707, + "ladak": 42312, + "ladakh": 45295, + "ladder": 16637, + "ladders": 47125, + "lade": 26447, + "laden": 28634, + "ladi": 12934, + "ladies": 28932, + "ladies": 3431, + "lads": 9803, + "lady": 7275, + "lady": 2909, + "ladybird": 43389, + "ladybug": 40038, + "ladygaga": 21232, + "laf": 47555, + "lafayette": 22683, + "lag": 30932, + "lag": 20394, + "laga": 30161, + "lage": 24369, + "lager": 36811, + "lager": 22989, + "lagh": 37237, + "laghate": 47565, + "laghateparth": 48780, + "lagi": 39786, + "lago": 42698, + "lago": 31476, + "lagoon": 22753, + "lagos": 12728, + "lagun": 18500, + "laguna": 23609, + "lah": 27315, + "lah": 4299, + "lahat": 42164, + "lahore": 16733, + "lai": 23947, + "laid": 42560, + "laid": 11160, + "lain": 46958, + "lain": 17151, + "laine": 35860, + "lair": 31981, + "lais": 34923, + "lak": 12890, + "lak": 26793, + "lake": 6441, + "lake": 2553, + "lakedistrict": 26437, + "lakel": 26133, + "lakeland": 34306, + "laker": 45717, + "lakers": 13570, + "lakes": 9265, + "lakeshore": 42595, + "lakeside": 30915, + "lakewood": 36417, + "lakh": 21487, + "lakhs": 37985, + "lakings": 34289, + "lakota": 45510, + "laksh": 24937, + "lakshmi": 39682, + "lal": 12301, + "lal": 19430, + "lala": 33661, + "lali": 21726, + "laliga": 32383, + "lam": 2022, + "lam": 5704, + "lama": 26049, + "lamar": 28678, + "lamar": 17284, + "lamb": 19863, + "lamb": 10034, + "lambda": 36687, + "lambert": 14574, + "lambeth": 43410, + "lambo": 45464, + "lamborgh": 18709, + "lamborghini": 19462, + "lambs": 30361, + "lame": 23192, + "lamin": 22337, + "laminated": 49079, + "lamo": 41461, + "lamont": 46719, + "lamp": 26700, + "lamp": 10725, + "lampard": 39989, + "lamps": 23424, + "lan": 1193, + "lan": 4872, + "lana": 15406, + "lanapar": 47437, + "lanaparrilla": 47819, + "lanc": 11872, + "lanca": 15694, + "lancashire": 20939, + "lancaster": 16446, + "lance": 26025, + "lance": 11609, + "lancer": 38195, + "lancers": 46392, + "lancia": 48698, + "lancs": 47540, + "land": 1567, + "land": 973, + "lande": 36556, + "landed": 9873, + "lander": 37247, + "lander": 9666, + "landers": 20019, + "landfall": 38465, + "landfill": 34947, + "landia": 41384, + "landing": 8292, + "landings": 46104, + "landlord": 28938, + "landlords": 35283, + "landmark": 15208, + "landmarks": 30393, + "lando": 25463, + "lando": 7065, + "landon": 32748, + "landrover": 38125, + "landry": 36137, + "lands": 40223, + "lands": 2961, + "landsc": 4384, + "landscape": 21123, + "landscape": 5727, + "landscapephotography": 28125, + "landscapes": 15344, + "landscaping": 25642, + "landslide": 31954, + "lane": 25534, + "lane": 3980, + "lanes": 10345, + "laney": 38552, + "lang": 7969, + "lang": 8578, + "lange": 32021, + "langford": 45615, + "langley": 28595, + "langu": 4095, + "language": 46103, + "language": 4781, + "languages": 13527, + "lani": 22964, + "lanka": 16221, + "lankan": 40531, + "lannister": 49056, + "lans": 43550, + "lansing": 30805, + "lant": 44504, + "lanta": 44768, + "lantern": 17185, + "lanterns": 33676, + "lantic": 32601, + "lantic": 27678, + "lants": 38425, + "lanyard": 46808, + "lao": 32475, + "lao": 29521, + "laos": 34353, + "lap": 7213, + "lap": 8639, + "lapd": 32557, + "lapel": 47961, + "lapland": 43633, + "laps": 18711, + "lapse": 33365, + "laptop": 10464, + "laptops": 32189, + "laq": 45026, + "lar": 1592, + "lar": 1652, + "lara": 19435, + "lard": 40347, + "lare": 22415, + "laredo": 48427, + "large": 40234, + "large": 3638, + "largely": 21418, + "larger": 12567, + "largest": 4960, + "largo": 44161, + "lari": 34676, + "lark": 43164, + "lark": 23536, + "larkin": 34769, + "larry": 18642, + "larry": 8242, + "lars": 8669, + "larsen": 39721, + "larson": 27973, + "larvae": 44840, + "las": 8295, + "las": 2552, + "lasag": 31210, + "lasagna": 40683, + "lasalle": 43866, + "laser": 25607, + "laser": 9885, + "lasers": 37060, + "lash": 31995, + "lash": 18480, + "lashes": 21015, + "lass": 24203, + "lass": 18263, + "lassic": 39430, + "last": 10600, + "last": 952, + "lasted": 25711, + "lasting": 13434, + "lastnight": 30159, + "lasts": 20141, + "lasvegas": 17789, + "lat": 1591, + "lat": 28437, + "lata": 47114, + "latam": 40012, + "late": 13267, + "late": 2325, + "latel": 49035, + "lately": 11824, + "latepost": 48328, + "later": 24109, + "later": 2941, + "lateral": 26646, + "latest": 46805, + "latest": 2053, + "latex": 27520, + "lati": 16357, + "latimes": 43356, + "latin": 16695, + "latin": 9888, + "latina": 27936, + "latino": 45734, + "latino": 19470, + "latinos": 40233, + "lation": 6191, + "latitude": 37392, + "lative": 15719, + "lator": 9291, + "lators": 28278, + "latt": 33561, + "latte": 17697, + "latter": 26198, + "latvia": 30034, + "lau": 1853, + "lau": 23090, + "lauderdale": 24352, + "laugh": 4969, + "laugh": 6332, + "laughed": 16746, + "laughing": 8301, + "laughs": 14322, + "laughter": 10722, + "laun": 2944, + "launch": 31168, + "launch": 2904, + "launched": 6125, + "launcher": 35782, + "launches": 7023, + "launching": 8565, + "laundering": 34079, + "laundry": 14797, + "laur": 15256, + "laura": 17091, + "laura": 7763, + "laure": 16932, + "laureate": 25675, + "laurel": 43370, + "laurel": 19942, + "lauren": 10456, + "lauren": 7634, + "laurence": 29353, + "laurent": 23226, + "laurie": 20326, + "laus": 38895, + "laus": 28111, + "lause": 22269, + "laut": 47688, + "lav": 13767, + "lav": 26919, + "lava": 16765, + "laven": 15047, + "lavender": 16033, + "laver": 28188, + "lavish": 35443, + "law": 2874, + "law": 2606, + "lawful": 33845, + "lawler": 47862, + "lawless": 39468, + "lawmaker": 37169, + "lawmakers": 21190, + "lawn": 31675, + "lawn": 11024, + "lawrence": 32221, + "lawrence": 8820, + "laws": 7306, + "lawson": 22152, + "lawsuit": 14346, + "lawsuits": 44331, + "lawyer": 10552, + "lawyers": 14232, + "lax": 17750, + "lax": 10024, + "lay": 7205, + "lay": 6360, + "laye": 25995, + "layer": 12411, + "layered": 28520, + "layers": 15900, + "laying": 12333, + "layla": 45050, + "layne": 48721, + "layo": 21738, + "layoffs": 29019, + "layout": 17314, + "lays": 19546, + "layton": 38061, + "laz": 18806, + "lazar": 33075, + "lazarus": 49126, + "laze": 41559, + "lazer": 43735, + "lazio": 33010, + "lazy": 32614, + "lazy": 10753, + "lb": 21958, + "lb": 7422, + "lbc": 37694, + "lbj": 45683, + "lbloggers": 48695, + "lbs": 8912, + "lc": 9584, + "lc": 7225, + "lcd": 21356, + "lcfc": 25339, + "lcs": 32279, + "ld": 1431, + "ld": 730, + "lder": 6945, + "lders": 43221, + "ldn": 37050, + "ldn": 2517, + "ldnont": 25827, + "ldnt": 21690, + "ldr": 37279, + "lds": 31235, + "le": 534, + "le": 579, + "lea": 2246, + "lea": 13324, + "leach": 35527, + "lead": 1328, + "lead": 2784, + "leader": 14806, + "leader": 3236, + "leaderboard": 34519, + "leaders": 3546, + "leadership": 36876, + "leadership": 3652, + "leading": 3833, + "leads": 5335, + "leaf": 9377, + "leaf": 7232, + "leaflet": 38289, + "leaflets": 39014, + "leafs": 16688, + "leafy": 42616, + "leagu": 13317, + "league": 16635, + "league": 2313, + "leagueof": 26022, + "leagueoflegends": 31737, + "leagues": 19888, + "leah": 24350, + "leah": 19308, + "leak": 42900, + "leak": 15489, + "leaked": 14353, + "leaking": 34097, + "leaks": 15657, + "leam": 39606, + "lean": 12447, + "lean": 8208, + "leaning": 24411, + "leanne": 41448, + "leans": 9357, + "leap": 29129, + "leap": 15392, + "leaps": 48080, + "lear": 1146, + "lear": 27663, + "learn": 16959, + "learn": 1768, + "learned": 6048, + "learnenglish": 49040, + "learner": 33547, + "learners": 19572, + "learning": 22632, + "learning": 2378, + "learns": 17569, + "learnt": 18959, + "leary": 36051, + "lease": 49041, + "lease": 14394, + "leased": 48352, + "leash": 36192, + "leasing": 29160, + "least": 3651, + "leather": 21417, + "leather": 5862, + "leau": 26498, + "leav": 3198, + "leave": 37512, + "leave": 3258, + "leaves": 5579, + "leaving": 5216, + "leban": 9360, + "lebanese": 23819, + "lebanon": 11695, + "leblanc": 46381, + "lebo": 44184, + "lebron": 11971, + "lebu": 47030, + "lec": 944, + "lec": 35374, + "leche": 46197, + "lect": 45392, + "lection": 18252, + "lections": 30995, + "lecture": 6617, + "lecturer": 23795, + "lectures": 21118, + "led": 8767, + "led": 912, + "ledge": 23647, + "ledge": 4815, + "ledger": 26817, + "leds": 36763, + "lee": 6224, + "lee": 2592, + "leed": 16483, + "leed": 40206, + "leeds": 38900, + "leeds": 7420, + "leek": 34585, + "leeminho": 37831, + "leen": 35311, + "leen": 15940, + "leep": 48875, + "leep": 10191, + "lees": 29324, + "lees": 34056, + "lef": 9152, + "left": 33949, + "left": 1823, + "leftist": 35143, + "lefto": 17437, + "leftover": 26414, + "leftovers": 28481, + "lefty": 33935, + "leg": 1211, + "leg": 4924, + "lega": 38674, + "legacy": 44108, + "legacy": 6447, + "legal": 17743, + "legal": 3998, + "legalization": 40584, + "legalize": 42921, + "legally": 14152, + "legate": 46009, + "lege": 8065, + "legen": 6105, + "legend": 5480, + "legend": 3539, + "legendary": 6053, + "legendof": 47915, + "legends": 6396, + "leges": 15356, + "legg": 18474, + "legg": 32511, + "legged": 25830, + "leggings": 22895, + "leggo": 43441, + "legi": 11183, + "legion": 35503, + "legion": 14525, + "legis": 7200, + "legislat": 16486, + "legislation": 14143, + "legislative": 16755, + "legislators": 31572, + "legislature": 22309, + "legit": 12563, + "legitim": 17656, + "legitimate": 24491, + "lego": 28117, + "lego": 7849, + "legos": 45359, + "legs": 7072, + "leh": 19105, + "leh": 29298, + "lehead": 28090, + "lehigh": 34527, + "lehman": 46094, + "lei": 15828, + "lei": 21830, + "leia": 32723, + "leic": 35073, + "leica": 30206, + "leice": 10026, + "leicester": 28795, + "leicester": 11510, + "leicestershire": 45358, + "leigh": 14849, + "leigh": 9292, + "leighton": 30782, + "leila": 41342, + "lein": 20026, + "lein": 28551, + "leinster": 32242, + "leip": 36401, + "leipzig": 41860, + "leis": 13133, + "leisure": 15849, + "leit": 35446, + "leith": 34141, + "lek": 26626, + "lek": 36535, + "lel": 46623, + "lele": 26075, + "lem": 10213, + "lem": 8428, + "leman": 24478, + "lemans": 26694, + "lement": 9693, + "lements": 15833, + "lemme": 23318, + "lemon": 12272, + "lemon": 7184, + "lemonade": 18884, + "lemons": 29576, + "lemore": 41147, + "len": 3687, + "len": 2159, + "lena": 22038, + "lend": 45397, + "lend": 24987, + "lender": 44734, + "lenders": 42443, + "lending": 20209, + "lene": 17628, + "leness": 36551, + "leng": 7861, + "length": 10130, + "lengths": 31858, + "lengthy": 32624, + "lenin": 41760, + "lennon": 18360, + "lennox": 45748, + "lenny": 48448, + "lenny": 30124, + "leno": 45357, + "lenovo": 25886, + "lens": 8666, + "lenses": 21264, + "lent": 20943, + "lent": 22605, + "lentil": 41511, + "lentils": 44269, + "leo": 24008, + "leo": 8312, + "leon": 6581, + "leon": 9763, + "leonard": 43849, + "leonard": 13142, + "leonardo": 20282, + "leone": 22864, + "leop": 11234, + "leopard": 15931, + "leopards": 40996, + "leopold": 45501, + "lep": 48884, + "leppard": 41656, + "lepre": 45641, + "ler": 5587, + "ler": 1803, + "lero": 15067, + "lerosis": 35455, + "leroy": 32441, + "lers": 6247, + "lery": 38184, + "les": 4339, + "les": 840, + "lesbian": 17419, + "lesbians": 43182, + "lesh": 32282, + "lesley": 25506, + "lesli": 13649, + "leslie": 16244, + "lesn": 39568, + "lesnar": 42223, + "less": 3242, + "less": 1285, + "lesser": 20369, + "lessly": 13103, + "lessness": 24847, + "lesson": 7714, + "lessons": 7199, + "lest": 24372, + "lest": 6794, + "lester": 23157, + "lester": 24023, + "lestwe": 29726, + "lestweforget": 30273, + "let": 1898, + "let": 1094, + "leta": 34319, + "lete": 34078, + "letes": 6815, + "leth": 30022, + "leth": 42462, + "lethal": 21905, + "lethbridge": 48390, + "leti": 34176, + "letics": 14504, + "letit": 46423, + "leto": 32203, + "leton": 37674, + "leton": 7462, + "lets": 10448, + "lets": 3243, + "letsgo": 16967, + "letsgo": 29789, + "letstalk": 35591, + "lett": 22428, + "lett": 9778, + "lette": 41798, + "lette": 10301, + "letter": 15567, + "letter": 4861, + "lettering": 26382, + "letterman": 38447, + "letters": 9181, + "letting": 9510, + "letto": 35449, + "lettu": 17933, + "lettuce": 18573, + "leu": 15691, + "leuke": 31031, + "leukemia": 32097, + "leum": 21571, + "leur": 45806, + "lev": 17022, + "lev": 29950, + "levan": 42543, + "leve": 36271, + "level": 21682, + "level": 2931, + "leveled": 48453, + "levels": 6295, + "leven": 44792, + "leven": 34729, + "lever": 20178, + "lever": 23094, + "leverage": 24030, + "leveraging": 37948, + "levi": 25630, + "levi": 19113, + "leviathan": 41736, + "levin": 36949, + "levine": 26594, + "levit": 22715, + "levy": 17147, + "lew": 5063, + "lew": 25329, + "lewan": 48349, + "lewd": 45241, + "lewes": 40431, + "lewi": 19589, + "lewis": 22043, + "lewis": 6020, + "lewisham": 37385, + "lewisham": 47633, + "lewishamilton": 42960, + "lewood": 37951, + "lex": 6586, + "lex": 9658, + "lexa": 48259, + "lexi": 44231, + "lexi": 24679, + "lexington": 22308, + "lexus": 20694, + "ley": 2565, + "ley": 1066, + "leye": 37061, + "leys": 45609, + "leys": 14834, + "leyton": 46573, + "lez": 26442, + "lf": 33960, + "lf": 22078, + "lfc": 37826, + "lfc": 8267, + "lfw": 28514, + "lg": 4546, + "lg": 11368, + "lga": 39348, + "lgb": 25401, + "lgbt": 11743, + "lgbt": 9592, + "lgbti": 42730, + "lgbtq": 47625, + "lgbtq": 14939, + "lgm": 39389, + "lh": 27794, + "lh": 31159, + "lhp": 45092, + "lhs": 33170, + "li": 554, + "li": 4250, + "lia": 26118, + "lia": 6964, + "liability": 29139, + "liaison": 39294, + "liam": 5258, + "liam": 7167, + "lian": 18058, + "liance": 40864, + "liar": 16334, + "liars": 23863, + "lias": 46021, + "lib": 10249, + "lib": 13345, + "libby": 36832, + "libdems": 40869, + "liber": 3425, + "liberal": 48032, + "liberal": 9985, + "liberalism": 40018, + "liberals": 15981, + "liberated": 38690, + "liberation": 19507, + "liberia": 32208, + "libertarian": 35067, + "liberties": 48623, + "liberty": 23397, + "liberty": 8480, + "libr": 2856, + "libra": 43038, + "librarian": 25148, + "librarians": 37806, + "libraries": 14277, + "library": 25713, + "library": 3519, + "libre": 49210, + "libre": 31681, + "libs": 26401, + "liby": 36390, + "libya": 16417, + "libyan": 42319, + "lic": 2508, + "lic": 3376, + "lice": 45691, + "licen": 6706, + "licence": 20550, + "license": 10337, + "licensed": 18752, + "licenses": 36414, + "licensing": 24219, + "lich": 23979, + "lich": 25875, + "lick": 29197, + "lick": 17541, + "licking": 33013, + "licks": 42117, + "lics": 44552, + "lid": 39369, + "lid": 17678, + "lidge": 45558, + "lido": 35683, + "lids": 41609, + "lie": 6570, + "lie": 2538, + "lieb": 45387, + "liebe": 37749, + "lied": 6486, + "lief": 38428, + "lien": 45716, + "lier": 3626, + "liers": 19303, + "lies": 37236, + "lies": 3205, + "liest": 14020, + "liet": 41107, + "lieu": 20401, + "lieu": 35313, + "lieutenant": 22538, + "lif": 16456, + "life": 2666, + "life": 970, + "lifeat": 27801, + "lifeboat": 37404, + "lifecycle": 49171, + "lifein": 48447, + "lifeis": 24824, + "lifeisgood": 46433, + "lifel": 15025, + "lifeline": 38438, + "lifelong": 21358, + "lifeof": 36061, + "lifesaving": 48016, + "lifespan": 49257, + "lifestyle": 46512, + "lifestyle": 7037, + "lifestyles": 48521, + "lifetime": 48737, + "lifetime": 9107, + "liff": 34404, + "liffe": 38942, + "lift": 33146, + "lift": 6779, + "lifted": 16783, + "lifter": 38555, + "lifting": 10857, + "lifts": 18291, + "lig": 19915, + "lig": 38493, + "liga": 16802, + "ligam": 31077, + "ligament": 48705, + "ligan": 27962, + "ligans": 42133, + "ligh": 7510, + "light": 3885, + "light": 1395, + "lighted": 18404, + "lighten": 32717, + "lightening": 28170, + "lighter": 14102, + "lighthouse": 13717, + "lighting": 5799, + "lightly": 26878, + "lightning": 7756, + "lightroom": 41454, + "lights": 3073, + "lightweight": 16278, + "ligu": 42920, + "ligue": 29196, + "lik": 4831, + "lik": 18495, + "like": 9175, + "like": 789, + "liked": 7112, + "likefor": 48444, + "likeli": 40666, + "likelihood": 48158, + "likely": 5256, + "liken": 36084, + "likes": 4724, + "liking": 16810, + "lil": 6012, + "lil": 4461, + "lilac": 33647, + "lili": 26686, + "lili": 48411, + "lilies": 38110, + "lillard": 47016, + "lille": 38705, + "lilli": 40920, + "lillian": 41563, + "lilly": 47825, + "lilly": 21815, + "lily": 23803, + "lily": 10647, + "lim": 2377, + "lim": 17204, + "lima": 17589, + "limb": 27061, + "limb": 32363, + "limbo": 46179, + "limbs": 34886, + "lime": 17385, + "lime": 11193, + "limel": 48658, + "limer": 16915, + "limerick": 19501, + "limestone": 27272, + "limit": 18933, + "limit": 9973, + "limitations": 32730, + "limited": 49229, + "limited": 3472, + "limiting": 35812, + "limitless": 35833, + "limits": 11966, + "limo": 33166, + "limous": 47287, + "limpopo": 47175, + "lin": 1254, + "lin": 2424, + "lina": 26110, + "lincol": 6239, + "lincoln": 16957, + "lincoln": 7454, + "lincolnshire": 29014, + "lind": 6492, + "linda": 45410, + "linda": 10760, + "linden": 44076, + "linden": 34832, + "lindo": 38467, + "lindsay": 29846, + "lindsay": 16858, + "lindsey": 29475, + "lindsey": 18128, + "line": 3674, + "line": 1148, + "linear": 19816, + "linebacker": 29848, + "lined": 11842, + "lineman": 31501, + "linen": 20032, + "liner": 11618, + "liners": 24463, + "lines": 3418, + "liness": 28633, + "lineup": 7316, + "lineups": 33589, + "ling": 4851, + "ling": 1358, + "linger": 29593, + "lingerie": 18473, + "lingering": 46494, + "lings": 11390, + "lington": 27673, + "lington": 9002, + "lingu": 34449, + "lingui": 29942, + "linguistic": 46847, + "linguistics": 48651, + "lining": 11589, + "link": 18433, + "link": 2468, + "linke": 15088, + "linked": 11059, + "linkedin": 16302, + "linkin": 40287, + "linkin": 49291, + "linking": 23296, + "links": 8113, + "linn": 37431, + "lino": 41189, + "lino": 34995, + "lins": 6567, + "linson": 15401, + "linton": 36479, + "linus": 49303, + "linux": 14061, + "lio": 19395, + "lion": 8872, + "lion": 5567, + "lionel": 19441, + "lions": 7093, + "lip": 8630, + "lip": 8546, + "lipo": 38795, + "lipp": 38074, + "lips": 8847, + "lipse": 10351, + "lipstick": 15618, + "liqu": 6310, + "lique": 32680, + "liqueur": 43612, + "liqui": 33817, + "liquid": 18366, + "liquid": 10158, + "liquidity": 42812, + "liquor": 17828, + "lis": 7297, + "lis": 12749, + "lisa": 25236, + "lisa": 7424, + "lisam": 43072, + "lisboa": 40052, + "lisbon": 17708, + "lish": 12658, + "lish": 2354, + "lished": 22620, + "lisle": 21529, + "lism": 34390, + "liss": 45489, + "liss": 35433, + "lisse": 49309, + "list": 1734, + "list": 1998, + "lista": 37812, + "listed": 6457, + "listen": 17454, + "listen": 2672, + "listened": 15347, + "listener": 34819, + "listeners": 26901, + "listening": 3656, + "listens": 25912, + "lister": 45109, + "listing": 8145, + "listings": 21987, + "liston": 48041, + "lists": 12281, + "lit": 2213, + "lit": 4350, + "lita": 30100, + "lite": 29273, + "lite": 13694, + "litecoin": 39063, + "liter": 3085, + "liter": 34904, + "literacy": 12841, + "literal": 24269, + "literally": 4719, + "literary": 13586, + "literature": 11072, + "litfest": 40369, + "lith": 37005, + "lithium": 22794, + "litho": 31088, + "lithograph": 49022, + "lithu": 21045, + "lithuania": 27068, + "liti": 24292, + "litigation": 31769, + "lito": 47381, + "litre": 25786, + "litres": 39919, + "litt": 1216, + "litt": 47583, + "litter": 45431, + "litter": 17118, + "litters": 45300, + "little": 7024, + "little": 1274, + "littlemix": 29731, + "littlest": 48969, + "litur": 36830, + "litz": 30357, + "liu": 20466, + "liv": 13895, + "liv": 19901, + "livan": 12785, + "live": 3215, + "live": 1064, + "lived": 8867, + "livel": 17973, + "liveli": 26566, + "livelihood": 46497, + "livelihoods": 47716, + "lively": 19663, + "liveme": 35396, + "livemusic": 15688, + "liven": 41057, + "liveon": 22815, + "livepd": 38742, + "livepd": 31899, + "liver": 4755, + "liver": 12639, + "liverpool": 29778, + "liverpool": 5366, + "livery": 23248, + "lives": 3247, + "livesmatter": 20348, + "livestock": 22079, + "livestream": 16844, + "livetweet": 38546, + "livin": 28061, + "living": 10965, + "living": 2815, + "livingston": 30551, + "lix": 45068, + "liz": 8632, + "liz": 12242, + "liza": 28787, + "lizard": 17221, + "lizards": 41991, + "lizasober": 44487, + "lizasoberano": 45076, + "lizz": 34430, + "lizzie": 29530, + "lizzy": 32306, + "lj": 34211, + "lj": 32273, + "lju": 44562, + "lk": 39110, + "lk": 26596, + "lka": 21881, + "ll": 1657, + "ll": 865, + "lla": 15419, + "llama": 36679, + "llan": 17281, + "llan": 38728, + "lland": 31150, + "llc": 17161, + "lle": 26550, + "lle": 29732, + "llen": 41197, + "ller": 7722, + "llers": 26426, + "lli": 47015, + "lli": 13368, + "llis": 25518, + "lll": 27177, + "llll": 34874, + "llll": 43485, + "llo": 19293, + "lloy": 10092, + "lloyd": 33339, + "lloyd": 12400, + "llp": 28042, + "lls": 40535, + "lly": 26379, + "lm": 6981, + "lm": 15282, + "lma": 4493, + "lmao": 5121, + "lmaoo": 32623, + "lmaooo": 33362, + "lmaoooo": 45232, + "lmfa": 8928, + "lmfao": 11068, + "lmfaooo": 47658, + "lmp": 43575, + "lms": 30381, + "ln": 31644, + "ln": 18654, + "lng": 22339, + "lnp": 39679, + "lo": 549, + "lo": 2982, + "loa": 39678, + "load": 4515, + "load": 2834, + "loaded": 6756, + "loader": 28492, + "loading": 9975, + "loads": 8691, + "loaf": 26467, + "loaf": 18273, + "loan": 28431, + "loan": 8176, + "loans": 14206, + "lob": 11197, + "lob": 46606, + "lobal": 34574, + "lobb": 27698, + "lobby": 12449, + "lobbying": 36047, + "lobe": 46325, + "lobes": 24148, + "lobo": 39323, + "lobos": 36586, + "lobster": 13793, + "loc": 1378, + "loc": 25826, + "local": 9202, + "local": 2029, + "localized": 49399, + "locally": 15603, + "locals": 15041, + "locate": 20490, + "located": 5677, + "location": 4372, + "locations": 9580, + "loch": 20188, + "loch": 14101, + "lock": 7201, + "lock": 4381, + "lockdown": 35636, + "locke": 29698, + "locked": 8371, + "locker": 14053, + "lockhart": 48642, + "lockheed": 36637, + "locking": 19978, + "locks": 13212, + "lockscreen": 42439, + "loco": 25555, + "locom": 22798, + "locomo": 46147, + "locomotive": 30439, + "locu": 33635, + "locust": 46237, + "lod": 45650, + "lodge": 10504, + "loe": 30113, + "loe": 25484, + "loeb": 49334, + "lof": 15011, + "loff": 31008, + "loft": 35707, + "loft": 20049, + "loftus": 46689, + "log": 3239, + "log": 7383, + "logan": 20655, + "logan": 10569, + "logans": 40752, + "logg": 43002, + "logged": 31457, + "logger": 39089, + "logging": 24444, + "logi": 3177, + "logia": 48031, + "logic": 10670, + "logical": 4791, + "logically": 24782, + "logie": 33445, + "logies": 7378, + "login": 31121, + "logist": 7407, + "logistics": 14755, + "logists": 12233, + "logne": 19911, + "logo": 31480, + "logo": 5750, + "logos": 24879, + "logs": 22745, + "logue": 27785, + "logy": 22721, + "logy": 1659, + "loh": 49129, + "loh": 37983, + "loi": 35128, + "loid": 31408, + "loin": 21760, + "loire": 46040, + "lois": 27040, + "lok": 19908, + "lok": 23575, + "loki": 24435, + "lol": 10721, + "lol": 1824, + "lola": 19065, + "lolita": 42615, + "lolla": 45483, + "lolli": 27906, + "lollipop": 34605, + "lolly": 48264, + "lolo": 16895, + "lolo": 37481, + "lolol": 25280, + "lololol": 34738, + "lolz": 35260, + "lom": 9279, + "loma": 42889, + "lombar": 25493, + "lombard": 46461, + "lombardi": 44346, + "lomond": 48941, + "lon": 1235, + "lon": 6507, + "london": 6835, + "london": 1789, + "londonmarathon": 35018, + "lone": 22220, + "lone": 13576, + "lonel": 28872, + "loneliness": 30310, + "lonely": 34509, + "lonely": 12368, + "lonelyplanet": 44984, + "long": 4792, + "long": 1538, + "longe": 25793, + "longer": 5349, + "longest": 10731, + "longevity": 35354, + "longh": 20286, + "longhorn": 41047, + "longhorns": 38295, + "longing": 38482, + "longlive": 47840, + "longs": 43618, + "longtime": 19685, + "loo": 731, + "loo": 11804, + "look": 8874, + "look": 1012, + "lookalike": 38307, + "lookbook": 39184, + "looked": 4913, + "lookin": 11254, + "looking": 36898, + "looking": 1312, + "lookout": 18330, + "looks": 1606, + "lool": 33125, + "loom": 37440, + "loom": 17199, + "looming": 35384, + "looms": 30550, + "loon": 28222, + "loona": 48137, + "looney": 45315, + "looo": 20902, + "loool": 36016, + "looool": 47038, + "looooo": 31484, + "loop": 19606, + "loop": 10408, + "loops": 21625, + "loos": 45723, + "loose": 43815, + "loose": 9786, + "loot": 21518, + "lop": 36734, + "lop": 17066, + "lopes": 49269, + "lopez": 12982, + "lor": 2179, + "lor": 11335, + "lord": 18896, + "lord": 3486, + "lorde": 35483, + "lords": 14969, + "lore": 12880, + "lore": 27218, + "loren": 13602, + "loren": 33398, + "lorenzo": 21342, + "lores": 34510, + "loretta": 40863, + "lori": 20164, + "lori": 23095, + "lorna": 46316, + "lorraine": 27602, + "lorry": 31354, + "los": 32217, + "los": 3087, + "losange": 14037, + "losangeles": 14638, + "lose": 43318, + "lose": 5354, + "loser": 18168, + "losers": 23201, + "loses": 14263, + "losing": 7918, + "loss": 34761, + "loss": 4327, + "losses": 16909, + "lost": 14258, + "lost": 2624, + "lostdog": 48482, + "lot": 5132, + "lot": 1954, + "loth": 43625, + "lothian": 31360, + "lothing": 42058, + "lotion": 25260, + "lotr": 34165, + "lots": 2958, + "lott": 42854, + "lotta": 29125, + "lotte": 16535, + "lotte": 7274, + "lottery": 16975, + "lottie": 48517, + "lotto": 28265, + "lotus": 13824, + "lou": 2207, + "lou": 9745, + "loubout": 38369, + "loud": 22884, + "loud": 7464, + "louder": 25904, + "loudest": 49214, + "loudly": 39256, + "lough": 21927, + "lough": 28045, + "loughborough": 49153, + "loui": 42173, + "louie": 25790, + "louis": 8916, + "louis": 4459, + "louisa": 40011, + "louise": 32275, + "louise": 13076, + "louisi": 12187, + "louisiana": 12946, + "louisville": 13860, + "louisvuitton": 44911, + "loun": 6466, + "lounge": 7141, + "lounging": 45430, + "lour": 29383, + "lourdes": 45071, + "louvre": 36995, + "lov": 8923, + "lov": 21229, + "lova": 37394, + "lovable": 38565, + "lovato": 18960, + "love": 2618, + "love": 793, + "lovecraft": 42405, + "loved": 3249, + "lovefl": 38884, + "loveher": 38306, + "lovehim": 45733, + "loveis": 30931, + "loveisland": 30970, + "loveislove": 43603, + "loveit": 24764, + "lovel": 8999, + "lovelies": 31412, + "lovelondon": 46493, + "lovely": 33250, + "lovely": 2165, + "lovemy": 20041, + "lovemyjob": 40130, + "loven": 33754, + "lover": 28508, + "lover": 7168, + "lovers": 48416, + "lovers": 5973, + "loves": 37773, + "loves": 3925, + "lovethe": 33040, + "lovethem": 48298, + "lovett": 47095, + "lovewins": 47687, + "loveyou": 39226, + "loveyou": 25964, + "loveyour": 26462, + "lovin": 33442, + "lovin": 16354, + "loving": 29568, + "loving": 3721, + "lovingly": 44100, + "low": 1049, + "low": 1042, + "loway": 16104, + "lowe": 17910, + "lowed": 22733, + "lowell": 24458, + "lower": 32578, + "lower": 4909, + "lowered": 34968, + "lowering": 35261, + "lowers": 36398, + "lowes": 38515, + "lowest": 12098, + "lowing": 8283, + "lowkey": 29481, + "lowry": 27444, + "lows": 4406, + "lox": 41725, + "loy": 4519, + "loy": 23929, + "loyal": 13032, + "loyalty": 14686, + "loyd": 44212, + "loyed": 29279, + "loyment": 18307, + "loyola": 32569, + "lp": 22282, + "lp": 6392, + "lpc": 44092, + "lpg": 47905, + "lpga": 34295, + "lps": 32094, + "lr": 20572, + "lr": 7041, + "lrt": 32996, + "ls": 19051, + "ls": 1268, + "lsd": 43766, + "lse": 46127, + "lse": 43886, + "lsu": 35428, + "lsu": 15672, + "lt": 13642, + "lt": 3333, + "ltc": 27664, + "ltd": 6802, + "lte": 25202, + "lton": 14237, + "lu": 664, + "lu": 9657, + "lub": 22469, + "lub": 11836, + "lubbock": 37660, + "lubric": 40963, + "luc": 7013, + "luc": 28014, + "luca": 21053, + "lucas": 23425, + "lucas": 10225, + "lucci": 45849, + "luce": 46217, + "lucent": 41552, + "lucer": 36042, + "luch": 36646, + "lucha": 38449, + "luci": 8787, + "lucia": 22290, + "luciano": 46365, + "lucid": 44540, + "lucie": 39461, + "lucifer": 46224, + "lucifer": 27687, + "lucille": 47454, + "lucin": 27523, + "luck": 9647, + "luck": 2820, + "luckiest": 42469, + "luckily": 20100, + "lucknow": 29407, + "lucky": 20495, + "lucky": 4133, + "lucrative": 41485, + "lucy": 17262, + "lucy": 10120, + "lud": 14288, + "lude": 28755, + "ludo": 40141, + "ludwig": 30633, + "lue": 45199, + "luf": 25264, + "lufc": 17818, + "luffy": 39047, + "lufthan": 37769, + "lufthansa": 39145, + "lug": 45521, + "lugg": 19673, + "luggage": 20138, + "luhan": 20975, + "luigi": 28444, + "luis": 25231, + "luis": 11339, + "luiz": 39633, + "lujah": 31639, + "luk": 21652, + "luka": 34878, + "lukaku": 37177, + "lukas": 37941, + "luke": 11970, + "luke": 5652, + "lul": 20861, + "lulla": 37019, + "lullaby": 41676, + "lulu": 32052, + "lulu": 26935, + "lum": 18112, + "lum": 5997, + "lumb": 36231, + "lumber": 27421, + "lumber": 34692, + "lumi": 41437, + "lumia": 31912, + "lumin": 15867, + "luminous": 37913, + "lump": 38704, + "lumpur": 34411, + "lun": 3221, + "lun": 49390, + "luna": 14425, + "lunar": 16043, + "lunatic": 45874, + "lunch": 10954, + "lunch": 2772, + "luncheon": 15104, + "lunches": 29705, + "lunchtime": 14330, + "lund": 30975, + "lund": 20181, + "lunes": 35648, + "lung": 38479, + "lung": 16271, + "lungs": 27366, + "lup": 27413, + "lupita": 49352, + "lupus": 36017, + "lur": 14439, + "lure": 31376, + "lures": 46747, + "lurking": 29941, + "lus": 7158, + "lusci": 38004, + "luscious": 39935, + "lush": 40382, + "lush": 16263, + "lust": 42071, + "lust": 12662, + "lustre": 46673, + "luther": 21848, + "luther": 17208, + "lutheran": 27341, + "luton": 28288, + "luv": 24726, + "luv": 8502, + "lux": 3439, + "lux": 16704, + "luxe": 26373, + "luxemb": 21314, + "luxembour": 22712, + "luxembourg": 23949, + "luxu": 16112, + "luxurious": 17292, + "luxury": 12083, + "luxury": 5247, + "luxurytravel": 29010, + "luz": 41008, + "lv": 10862, + "lv": 11184, + "lvl": 31256, + "lw": 40515, + "lw": 35115, + "lx": 30789, + "ly": 1251, + "ly": 597, + "lydia": 24316, + "lyf": 43688, + "lyfe": 30787, + "lyft": 32944, + "lying": 7175, + "lyk": 46376, + "lyle": 36828, + "lym": 20087, + "lyme": 31167, + "lymph": 30073, + "lymphoma": 37648, + "lyn": 3957, + "lyn": 5054, + "lynch": 31586, + "lynch": 13560, + "lynd": 33416, + "lynda": 42959, + "lyndon": 48518, + "lynn": 25303, + "lynn": 10667, + "lynne": 26900, + "lynx": 28941, + "lyon": 17176, + "lyons": 29453, + "lyric": 24366, + "lyric": 21291, + "lyrical": 33358, + "lyricist": 49013, + "lyrics": 9551, + "lyrix": 46814, + "lys": 45054, + "lyte": 40059, + "lywood": 4012, + "lz": 30818, + "lé": 39641, + "m": 76, + "m": 332, + "ma": 577, + "ma": 1226, + "maa": 42774, + "maa": 21555, + "maan": 33668, + "maar": 48927, + "maas": 43332, + "mab": 35639, + "mabel": 47319, + "mable": 23001, + "mably": 40082, + "mabu": 44682, + "mac": 1961, + "mac": 4945, + "macar": 21558, + "macaroni": 41824, + "macarthur": 36785, + "macau": 43984, + "macau": 33370, + "macbeth": 36321, + "macbook": 20617, + "macdonald": 20315, + "mace": 44869, + "maced": 21102, + "macedonia": 27071, + "macfar": 45374, + "macfarlane": 48825, + "mach": 2637, + "mach": 35091, + "machado": 42318, + "mache": 43220, + "macher": 29330, + "machi": 41783, + "machin": 17972, + "machine": 11539, + "machine": 4169, + "machinelearning": 13621, + "machinery": 21858, + "machines": 11108, + "machining": 45562, + "macho": 43977, + "macht": 45225, + "macin": 36533, + "mack": 8590, + "mack": 12145, + "mackay": 32497, + "macken": 48057, + "mackenzie": 22351, + "mackerel": 35002, + "mackin": 26010, + "macklemore": 41758, + "macle": 33843, + "maclean": 47137, + "macleod": 43684, + "macmillan": 36364, + "macmillan": 35191, + "macon": 35818, + "macos": 45469, + "macqu": 38365, + "macquarie": 40858, + "macro": 20891, + "macro": 16626, + "macron": 24859, + "macs": 46548, + "macy": 17113, + "macys": 47652, + "mad": 2740, + "mad": 3843, + "mada": 37799, + "madagas": 24758, + "madagascar": 25744, + "madam": 33634, + "madam": 27538, + "madame": 23507, + "madd": 31717, + "madden": 19093, + "maddie": 39959, + "maddie": 18875, + "maddow": 32644, + "maddy": 31734, + "made": 5388, + "made": 1105, + "madein": 13670, + "madeira": 33810, + "madel": 34532, + "madele": 29831, + "madeleine": 33264, + "madeline": 33905, + "madewith": 28627, + "madewithunity": 43190, + "madhu": 23000, + "madhuri": 38346, + "madhuridixit": 43889, + "madhya": 48302, + "madi": 6527, + "madi": 27282, + "madison": 24798, + "madison": 8791, + "madmen": 45452, + "madness": 8755, + "madon": 44852, + "madonna": 14137, + "madra": 27416, + "madras": 42046, + "madre": 42130, + "madri": 5529, + "madrid": 5909, + "mads": 41201, + "madu": 34913, + "madurai": 49159, + "maduro": 32912, + "mae": 16898, + "mae": 17339, + "maer": 47088, + "maestro": 24140, + "mafi": 47164, + "mafia": 14890, + "mag": 1191, + "mag": 4508, + "maga": 8694, + "magaz": 2974, + "magazine": 3113, + "magazines": 22253, + "magdal": 29673, + "mage": 46568, + "mage": 10923, + "magee": 43872, + "magenta": 38091, + "magento": 42442, + "mages": 31059, + "maggi": 29611, + "maggie": 41443, + "maggie": 14524, + "maggio": 49087, + "magh": 45555, + "magi": 19270, + "magic": 13061, + "magic": 3778, + "magical": 36408, + "magical": 7823, + "magician": 26368, + "magin": 42678, + "maging": 41310, + "magn": 10290, + "magna": 34076, + "magne": 9921, + "magnesium": 36379, + "magnet": 18240, + "magnetic": 13838, + "magnets": 33030, + "magni": 24297, + "magnific": 9725, + "magnificent": 10724, + "magnitude": 22955, + "magno": 21184, + "magnolia": 27123, + "magnu": 45198, + "magnum": 23496, + "magnus": 26275, + "magpie": 45973, + "mags": 31021, + "maguire": 26470, + "mah": 7206, + "mah": 10801, + "maha": 12237, + "maha": 33983, + "mahal": 22301, + "mahan": 45191, + "mahar": 11635, + "maharaj": 38488, + "maharashtra": 19328, + "mahat": 32434, + "mahatma": 40530, + "mahe": 15756, + "maher": 29826, + "mahesh": 33448, + "mahesh": 22095, + "mahi": 32529, + "mahi": 38659, + "mahin": 24113, + "mahindra": 31285, + "mahmoud": 41361, + "mahog": 30804, + "mahogany": 33084, + "mahon": 45864, + "mahon": 20371, + "mahone": 26634, + "mai": 7138, + "mai": 14595, + "maia": 46585, + "maid": 23148, + "maid": 10226, + "maidan": 37346, + "maiden": 37011, + "maiden": 13809, + "maids": 27305, + "maidstone": 44395, + "mail": 10478, + "mail": 2614, + "mailbox": 31482, + "mailed": 42314, + "mailing": 26680, + "mailonline": 26021, + "mails": 45213, + "main": 3904, + "main": 2623, + "maine": 18639, + "maine": 7836, + "mained": 15609, + "mainedcm": 15845, + "mainland": 27629, + "mainly": 15280, + "mains": 33656, + "mainst": 42102, + "mainstream": 18034, + "maintain": 12954, + "maintained": 26665, + "maintaining": 21964, + "maintains": 38335, + "mainten": 9399, + "maintenance": 9610, + "mais": 28153, + "maisie": 47355, + "maison": 37065, + "maison": 27626, + "mait": 26387, + "maize": 35386, + "maj": 2948, + "maj": 28723, + "maja": 47498, + "maje": 9852, + "majestic": 15335, + "majesty": 21188, + "major": 8008, + "major": 3350, + "majority": 10508, + "majors": 23597, + "mak": 11271, + "mak": 19253, + "makar": 42242, + "makati": 39402, + "make": 3232, + "make": 1078, + "makeaw": 45859, + "makeinindia": 42739, + "makeit": 26308, + "maken": 47093, + "makeover": 17926, + "maker": 15196, + "maker": 4836, + "makers": 6577, + "makerspace": 42400, + "makes": 2088, + "makeshift": 43274, + "makeu": 41707, + "makeup": 26402, + "makeup": 5853, + "makeyourown": 34090, + "makeyourownlane": 34823, + "maki": 34514, + "makin": 43096, + "makin": 22407, + "making": 17976, + "making": 1665, + "makk": 39852, + "maknae": 44118, + "mako": 49061, + "mal": 1662, + "mal": 3796, + "mala": 28290, + "malade": 36928, + "malaga": 35395, + "malala": 41137, + "malam": 48956, + "malaria": 24929, + "malawi": 23405, + "malay": 5323, + "malay": 42430, + "malayalam": 34860, + "malaysi": 39668, + "malaysia": 8146, + "malaysian": 21136, + "malbec": 47741, + "malcol": 12645, + "malcolm": 14139, + "maldives": 16795, + "male": 11326, + "male": 2801, + "males": 14426, + "malhotra": 28866, + "mali": 6701, + "mali": 22669, + "malia": 46714, + "malibu": 21723, + "malicious": 42147, + "malign": 41122, + "malik": 11394, + "mall": 10984, + "mall": 6220, + "mallorca": 28082, + "mallory": 38968, + "malls": 36447, + "malm": 44071, + "malnutrition": 41153, + "malo": 43518, + "malone": 19852, + "maloney": 45897, + "mals": 25370, + "malt": 21688, + "malta": 16989, + "maltese": 39838, + "malvern": 39356, + "malware": 24153, + "mam": 4404, + "mam": 17778, + "mama": 7133, + "mamamoo": 36012, + "mamas": 42395, + "mamba": 44189, + "mament": 45690, + "mami": 43858, + "mamma": 34893, + "mammal": 33385, + "mammals": 31987, + "mammoth": 28022, + "man": 723, + "man": 786, + "mana": 29467, + "mana": 15837, + "manafort": 40108, + "manag": 1830, + "manage": 9770, + "managed": 7928, + "management": 3319, + "manager": 3898, + "managerial": 44261, + "managers": 12853, + "manages": 29699, + "managing": 10892, + "manas": 44188, + "manatee": 46558, + "mance": 2324, + "manchester": 24424, + "manchester": 4651, + "mancini": 47681, + "mancity": 31538, + "mancrush": 36945, + "mancrushmonday": 39307, + "mand": 4325, + "mand": 27244, + "mandala": 41106, + "mandarin": 26455, + "mandate": 26228, + "mandatory": 19934, + "mandel": 34960, + "mandela": 16280, + "mandi": 38961, + "mandir": 35815, + "mando": 34006, + "mands": 12340, + "mandu": 31440, + "mandy": 41505, + "mandy": 24302, + "mane": 44471, + "mane": 16044, + "maneu": 33216, + "mang": 25616, + "mang": 31096, + "manga": 11873, + "mangal": 43027, + "manger": 48251, + "mango": 43831, + "mango": 13962, + "mangrove": 47180, + "manhatt": 10152, + "manhattan": 10961, + "mani": 5654, + "mani": 10718, + "mania": 8435, + "maniac": 31814, + "maniacs": 41444, + "manian": 40077, + "manic": 23017, + "manic": 37825, + "manicure": 33637, + "manife": 14379, + "manifest": 34422, + "manifestation": 48348, + "manifesto": 20907, + "manil": 38827, + "manila": 10969, + "manipu": 40261, + "manipul": 19237, + "manipulation": 30277, + "manipur": 47757, + "manish": 41759, + "manish": 44720, + "manit": 15693, + "manitoba": 20342, + "manjaro": 41489, + "mankind": 24155, + "manly": 25194, + "mann": 19396, + "mann": 4783, + "manne": 30160, + "manned": 26139, + "mannequin": 43388, + "manner": 20700, + "manners": 31693, + "manning": 15996, + "manny": 37054, + "manny": 20933, + "mano": 15753, + "mano": 24016, + "manoj": 41146, + "manor": 41830, + "manor": 13614, + "mans": 28422, + "mans": 7746, + "mansfield": 25543, + "manship": 15460, + "mansion": 13404, + "manslaughter": 48632, + "manson": 26715, + "mant": 25122, + "mant": 27037, + "manta": 41431, + "mantis": 39946, + "mantle": 22159, + "mantra": 25162, + "manu": 3404, + "manu": 25799, + "manual": 12268, + "manuel": 29171, + "manuel": 9567, + "manufac": 5105, + "manufacture": 27741, + "manufactured": 24010, + "manufacturer": 15668, + "manufacturers": 18763, + "manufacturing": 8386, + "manure": 47907, + "manus": 28181, + "manuscript": 24365, + "manuscripts": 40765, + "manutd": 20994, + "many": 28484, + "many": 1346, + "manziel": 40637, + "mao": 47447, + "mao": 25605, + "maori": 43400, + "map": 25180, + "map": 3923, + "maple": 21980, + "maple": 10570, + "mapleleafs": 41257, + "mapoli": 28768, + "mapp": 36894, + "mapped": 41596, + "mapping": 15231, + "maps": 8765, + "mapu": 42082, + "mar": 675, + "mar": 3091, + "mara": 15655, + "marais": 47913, + "maran": 44732, + "marath": 16274, + "marathi": 34102, + "marathon": 40764, + "marathon": 5910, + "marau": 38475, + "marbella": 36182, + "marble": 45429, + "marble": 13071, + "marbles": 42931, + "marc": 14054, + "marc": 9075, + "marca": 38242, + "marcel": 17726, + "marcel": 24652, + "marcelo": 35939, + "march": 10638, + "march": 2227, + "marche": 36173, + "marched": 37976, + "marches": 38249, + "marchfor": 31721, + "marching": 15082, + "marchmadness": 28555, + "marci": 36698, + "marcia": 41075, + "marck": 47733, + "marco": 24719, + "marco": 10924, + "marcor": 39945, + "marcorubio": 41143, + "marcos": 21696, + "marcu": 20760, + "marcus": 48955, + "marcus": 9895, + "mardi": 39728, + "mardi": 29229, + "mardigras": 43343, + "mare": 26512, + "mare": 8870, + "mares": 19724, + "marg": 44014, + "margar": 16838, + "margare": 10232, + "margaret": 12185, + "margarita": 25958, + "margaritas": 42679, + "margate": 37428, + "margin": 19464, + "margin": 21357, + "marginal": 38320, + "margins": 33763, + "margot": 37144, + "mari": 2603, + "mari": 19322, + "maria": 41109, + "maria": 6595, + "mariachi": 44299, + "mariah": 31214, + "mariah": 24789, + "mariahcarey": 36538, + "marian": 41129, + "marian": 24677, + "mariana": 44224, + "marianne": 32214, + "mariano": 43988, + "marie": 20657, + "marie": 7864, + "marietta": 46634, + "marig": 41002, + "marijuana": 9864, + "maril": 14611, + "marilyn": 38959, + "marilyn": 18489, + "marin": 8910, + "marin": 23992, + "marina": 12060, + "marinated": 33406, + "marine": 20674, + "marine": 5746, + "mariner": 39972, + "mariners": 19086, + "marines": 15018, + "marino": 30878, + "mario": 39176, + "mario": 7600, + "marion": 37765, + "marion": 18397, + "maris": 21512, + "maris": 33093, + "marisa": 42938, + "mariska": 44703, + "marissa": 31219, + "marist": 48223, + "mariti": 13124, + "maritime": 14331, + "marj": 38639, + "mark": 3805, + "mark": 2110, + "marke": 2399, + "marked": 12360, + "marker": 18170, + "markers": 23664, + "market": 11614, + "market": 2196, + "marketer": 33482, + "marketers": 23682, + "marketing": 19535, + "marketing": 2905, + "marketplace": 18241, + "markets": 7292, + "markham": 39817, + "marking": 14705, + "markings": 41046, + "markle": 32672, + "marko": 38338, + "marks": 5466, + "markus": 33725, + "marl": 24922, + "marlborough": 43515, + "marlene": 45117, + "marley": 16504, + "marlin": 34275, + "marlins": 23309, + "marlon": 32995, + "marmalade": 39068, + "marnock": 48305, + "maro": 27029, + "maroon": 20501, + "marqu": 20704, + "marque": 13012, + "marquee": 27725, + "marquette": 37624, + "marquez": 27317, + "marquis": 33530, + "marr": 32871, + "marrake": 37125, + "marrakech": 39006, + "marri": 3839, + "marriage": 38047, + "marriage": 7040, + "marriages": 38190, + "married": 6791, + "marries": 46283, + "marriott": 19211, + "marrow": 31030, + "marry": 13288, + "marrying": 40507, + "mars": 41469, + "mars": 7496, + "marsden": 43344, + "marse": 26577, + "marseille": 30365, + "marsh": 9237, + "marsh": 13505, + "marsha": 21491, + "marshal": 26608, + "marshall": 30939, + "marshall": 9811, + "marshals": 44175, + "marshes": 43450, + "marshmal": 21069, + "marshmallow": 28530, + "marshmallows": 39471, + "mart": 2348, + "mart": 7772, + "marta": 32858, + "martens": 43211, + "marth": 34493, + "martha": 16427, + "marti": 20577, + "martial": 17088, + "martialarts": 35895, + "martian": 30214, + "martin": 6929, + "martin": 3690, + "martina": 34393, + "martinez": 13913, + "marting": 47570, + "martini": 22199, + "martino": 41675, + "martins": 30569, + "marty": 9926, + "marty": 17169, + "martyn": 44075, + "martyr": 36155, + "martyr": 26067, + "martyrdom": 43110, + "martyred": 39114, + "martyrs": 24707, + "maru": 37413, + "maru": 31838, + "marvel": 13835, + "marvel": 5996, + "marvelcomics": 46897, + "marvell": 26576, + "marvellous": 28402, + "marvelous": 25487, + "marvin": 19675, + "marx": 30559, + "marx": 26001, + "marxist": 45205, + "mary": 5146, + "mary": 2676, + "maryam": 33636, + "maryam": 36393, + "maryland": 11379, + "marys": 40905, + "marys": 40228, + "mas": 5226, + "mas": 1412, + "masa": 24995, + "masa": 41868, + "masala": 31483, + "masc": 23564, + "mascar": 46984, + "mascara": 31635, + "mascot": 13983, + "mascots": 43266, + "mascul": 25589, + "masculine": 48269, + "masculinity": 40465, + "mase": 49128, + "maser": 25798, + "maserati": 30442, + "mash": 12317, + "mash": 15680, + "mashable": 41026, + "mashed": 27395, + "mashup": 27079, + "masi": 35965, + "masjid": 31420, + "mask": 19262, + "mask": 8306, + "masked": 25757, + "masking": 47046, + "masks": 19055, + "maslow": 44359, + "mason": 17424, + "mason": 9699, + "masonic": 36491, + "masonry": 30764, + "masons": 37195, + "masqu": 26593, + "masquer": 29604, + "masquerade": 36944, + "mass": 4636, + "mass": 4854, + "massach": 14484, + "massachuse": 14577, + "massachusetts": 14756, + "massacre": 14696, + "massage": 13055, + "masse": 41735, + "masses": 22978, + "massey": 29868, + "massi": 17239, + "massimo": 45821, + "massive": 4818, + "massively": 34297, + "mast": 45916, + "mast": 27920, + "master": 4534, + "master": 3498, + "mastercard": 40542, + "masterchef": 34809, + "masterclass": 17529, + "mastered": 32616, + "masterful": 46823, + "mastering": 28326, + "mastermind": 34029, + "masterpiece": 12066, + "masterpieces": 37596, + "masters": 6913, + "mastery": 34800, + "mastiff": 42311, + "maswar": 47887, + "mat": 905, + "mat": 9063, + "mata": 17270, + "match": 7733, + "match": 2439, + "matcha": 32433, + "matchday": 15947, + "matched": 17792, + "matches": 8609, + "matching": 11840, + "matchup": 19355, + "matchups": 49162, + "mate": 6137, + "mate": 2936, + "mated": 33813, + "mateo": 34991, + "mater": 23724, + "materi": 7084, + "material": 7118, + "materials": 8161, + "maternal": 26131, + "maternity": 23894, + "mates": 5817, + "math": 13277, + "math": 6025, + "mathe": 8725, + "mathemat": 11901, + "mathematical": 25609, + "mathematician": 41036, + "mathematics": 20113, + "mathew": 36333, + "mathews": 37120, + "mathi": 23014, + "mathieu": 40417, + "maths": 14763, + "mati": 12716, + "mati": 32268, + "matic": 36859, + "matic": 7900, + "matically": 38282, + "matics": 23634, + "matil": 26751, + "matilda": 36308, + "matin": 44849, + "matinee": 38525, + "mating": 34346, + "mation": 11701, + "matisse": 43446, + "mato": 13127, + "matologist": 48842, + "matology": 27940, + "matory": 25519, + "matri": 27041, + "matrix": 18078, + "mats": 22259, + "matsu": 30242, + "matt": 7972, + "matt": 3972, + "mattb": 42791, + "matte": 31237, + "matte": 19771, + "mattel": 35365, + "matteo": 33120, + "matter": 30471, + "matter": 3828, + "matters": 5708, + "matth": 41846, + "matthe": 5116, + "matthew": 17588, + "matthew": 7008, + "matthews": 16739, + "matthi": 29853, + "matthias": 45104, + "matti": 39840, + "mattress": 23438, + "matty": 31233, + "matty": 29176, + "matu": 40616, + "matur": 22897, + "mature": 14417, + "maturity": 28047, + "mau": 8134, + "mau": 23033, + "maui": 20463, + "maul": 30725, + "maur": 10574, + "maure": 25191, + "maureen": 31723, + "maurice": 20200, + "mauricio": 39066, + "mauriti": 28406, + "mauritius": 29305, + "mauro": 41691, + "mav": 25697, + "maver": 16700, + "maverick": 27425, + "mavericks": 30092, + "mavs": 30665, + "maw": 39351, + "maw": 42271, + "mawards": 37682, + "max": 4898, + "max": 3902, + "maxi": 8554, + "maxi": 23266, + "maxim": 19892, + "maxim": 38574, + "maximize": 28673, + "maximum": 13162, + "maximus": 44312, + "maxine": 38468, + "maxwell": 19611, + "maxx": 37466, + "may": 1686, + "may": 1270, + "maya": 45783, + "maya": 12987, + "mayan": 37952, + "maybe": 3746, + "mayday": 29957, + "mayer": 21196, + "mayfair": 35171, + "mayfield": 33933, + "mayhem": 21502, + "maymay": 26600, + "maymay": 33853, + "maymayentrata": 30480, + "maynard": 32487, + "mayne": 35771, + "mayo": 22449, + "mayo": 11280, + "mayor": 15429, + "mayor": 4676, + "mayoral": 28983, + "mayorof": 43533, + "mayors": 28501, + "mays": 35445, + "maythe": 42281, + "mayward": 45751, + "mayward": 23519, + "mayweather": 22774, + "maz": 9177, + "maz": 36215, + "mazda": 18506, + "maze": 21988, + "mazz": 29439, + "mañ": 37059, + "mañana": 39354, + "mb": 758, + "mb": 3996, + "mba": 8329, + "mban": 46685, + "mbar": 44452, + "mbb": 10736, + "mbc": 20137, + "mbe": 38395, + "mbe": 27004, + "mber": 5467, + "mber": 1034, + "mberg": 26372, + "mbers": 5443, + "mbi": 45347, + "mble": 20310, + "mble": 4756, + "mbles": 28693, + "mbling": 28604, + "mbo": 25733, + "mbo": 11319, + "mbps": 44896, + "mbs": 10370, + "mbta": 38979, + "mbu": 42228, + "mbuhari": 36752, + "mc": 1278, + "mc": 4126, + "mca": 40570, + "mca": 14635, + "mcal": 28663, + "mcar": 43776, + "mcbride": 35080, + "mcc": 21192, + "mccabe": 37628, + "mccaf": 47385, + "mccain": 20397, + "mccall": 34844, + "mccann": 27140, + "mccar": 9570, + "mccarthy": 16974, + "mccartney": 19958, + "mccl": 24709, + "mccla": 43672, + "mccle": 40139, + "mcclure": 44945, + "mcco": 46152, + "mccon": 32638, + "mccor": 23057, + "mccormack": 45164, + "mccormick": 39088, + "mccoy": 20218, + "mccr": 41996, + "mccre": 25393, + "mccul": 38833, + "mccull": 41782, + "mcd": 28930, + "mcder": 27355, + "mcdermott": 34504, + "mcdon": 12171, + "mcdonald": 10741, + "mcdonalds": 17674, + "mcdonnell": 34360, + "mcdowell": 34119, + "mce": 26864, + "mcel": 28752, + "mcen": 47423, + "mcfad": 36976, + "mcfadden": 42105, + "mcfar": 29020, + "mcfarlane": 47174, + "mcfc": 16416, + "mcfly": 38211, + "mcg": 42507, + "mcg": 27995, + "mcgee": 29223, + "mcgill": 46524, + "mcgill": 35511, + "mcgin": 29596, + "mcgowan": 40462, + "mcgr": 25169, + "mcgra": 29367, + "mcgrath": 28759, + "mcgraw": 40950, + "mcgregor": 19642, + "mcgu": 34294, + "mcguinness": 45299, + "mcguire": 32635, + "mci": 46212, + "mci": 45491, + "mcil": 30481, + "mcin": 18770, + "mcintosh": 45353, + "mcintyre": 33369, + "mck": 6781, + "mckay": 33611, + "mcke": 27424, + "mckee": 43529, + "mcken": 42619, + "mckenna": 24924, + "mckenzie": 25502, + "mckin": 15437, + "mckinley": 39891, + "mckinney": 33554, + "mckinnon": 48736, + "mckinsey": 48143, + "mcl": 49021, + "mcla": 12565, + "mclaren": 37381, + "mclaren": 16789, + "mclau": 32285, + "mclaughlin": 35346, + "mcle": 25299, + "mclean": 28666, + "mcleod": 40259, + "mcm": 12251, + "mcmahon": 24026, + "mcmaster": 42703, + "mcmillan": 45603, + "mcn": 42919, + "mcnam": 32682, + "mcnamara": 37506, + "mcne": 42545, + "mco": 33723, + "mcqueen": 22544, + "mcr": 29884, + "mcr": 16966, + "mcs": 27020, + "mcu": 30403, + "md": 8637, + "md": 4732, + "mdc": 38773, + "mdc": 41761, + "mds": 48746, + "mdt": 40822, + "me": 613, + "me": 614, + "mea": 46045, + "mea": 17711, + "mead": 12134, + "mead": 21567, + "meade": 37218, + "meado": 16402, + "meadow": 25213, + "meadow": 17195, + "meadows": 17178, + "meal": 29662, + "meal": 5478, + "meals": 11229, + "mean": 4189, + "mean": 3450, + "meand": 48015, + "meaning": 14586, + "meaning": 8342, + "meaningful": 17480, + "meaningless": 48932, + "meanings": 45814, + "means": 3494, + "meant": 8674, + "meantime": 27499, + "meanwhile": 9650, + "meas": 5867, + "measles": 38230, + "measurable": 48010, + "measure": 15261, + "measure": 10579, + "measured": 23154, + "measurement": 20973, + "measurements": 29894, + "measures": 11936, + "measuring": 18064, + "meat": 10805, + "meat": 6480, + "meatball": 43642, + "meatballs": 29233, + "meath": 37920, + "meatless": 48085, + "meats": 29558, + "mec": 27432, + "mecca": 36095, + "mech": 38305, + "mechan": 6715, + "mechanic": 24582, + "mechanical": 14467, + "mechanics": 20536, + "mechanism": 22576, + "mechanisms": 28610, + "meck": 41908, + "med": 1948, + "med": 2177, + "meda": 33614, + "medal": 29714, + "medal": 6974, + "medalist": 21040, + "medalists": 43397, + "medalli": 31349, + "medallion": 43469, + "medallist": 41472, + "medals": 14710, + "mede": 48225, + "meded": 27627, + "medi": 1436, + "media": 22064, + "media": 1895, + "mediac": 37490, + "median": 30491, + "mediation": 42829, + "medic": 3602, + "medic": 35441, + "medicaid": 25421, + "medical": 18432, + "medical": 4116, + "medicare": 23710, + "medication": 23771, + "medications": 37181, + "medicinal": 28772, + "medicine": 5616, + "medicines": 26541, + "medics": 46688, + "medieval": 38956, + "medieval": 10789, + "medina": 27281, + "mediocre": 41170, + "medit": 19130, + "meditate": 38039, + "meditation": 10827, + "mediter": 14194, + "mediterran": 14358, + "mediterranean": 15327, + "medium": 8675, + "medley": 24793, + "meds": 25075, + "medtech": 42044, + "medusa": 44216, + "medway": 42286, + "mee": 1725, + "mee": 14075, + "meek": 28935, + "meen": 37940, + "meen": 46515, + "meer": 26714, + "meer": 27555, + "meet": 5714, + "meet": 1633, + "meeting": 48566, + "meeting": 2071, + "meetings": 9980, + "meets": 5972, + "meetthe": 27575, + "meetup": 15430, + "meg": 11500, + "meg": 16186, + "mega": 15979, + "mega": 9068, + "megab": 38103, + "megadeth": 46741, + "megal": 37650, + "megam": 26073, + "megan": 19127, + "megan": 11503, + "megap": 33624, + "megat": 35581, + "megh": 31192, + "meghan": 39939, + "meghan": 18261, + "meh": 10512, + "meh": 22211, + "mehta": 25031, + "mei": 22564, + "mei": 25198, + "meier": 29812, + "mein": 28857, + "mein": 21466, + "meister": 28407, + "mek": 44645, + "mel": 1902, + "mel": 6834, + "mela": 35032, + "melan": 22261, + "melanch": 44818, + "melancholy": 47821, + "melani": 34031, + "melania": 32796, + "melanie": 22153, + "melanoma": 40862, + "melb": 47007, + "melb": 28980, + "melbourne": 28387, + "melbourne": 6995, + "melee": 45108, + "meli": 28885, + "melinda": 46303, + "melis": 18913, + "melissa": 41866, + "melissa": 13030, + "mell": 22531, + "mell": 41583, + "mello": 47594, + "mellon": 45162, + "mellow": 32034, + "melo": 10354, + "melo": 22374, + "melodic": 41877, + "melodies": 38412, + "melody": 19119, + "melon": 12146, + "melrose": 36296, + "melt": 22209, + "melt": 15957, + "meltdown": 30613, + "melted": 23037, + "melting": 19247, + "melton": 46062, + "melts": 31446, + "melville": 46030, + "melvin": 31544, + "mely": 6373, + "mem": 4937, + "mem": 34944, + "memb": 2114, + "member": 29566, + "member": 1640, + "members": 2567, + "membership": 11562, + "membrane": 34088, + "meme": 35157, + "meme": 9169, + "memes": 12828, + "memo": 15967, + "memo": 19334, + "memoir": 20532, + "memoirs": 45311, + "memor": 1858, + "memorab": 26271, + "memorabilia": 27488, + "memorable": 13172, + "memorial": 16285, + "memorial": 4642, + "memorialday": 21598, + "memoriam": 48191, + "memories": 4304, + "memory": 44766, + "memory": 5137, + "memph": 10285, + "memphis": 38432, + "memphis": 11298, + "men": 1552, + "men": 1656, + "mena": 23052, + "menace": 29949, + "mend": 8151, + "mend": 46927, + "mendel": 49268, + "mendes": 18060, + "mendez": 48275, + "mendo": 19327, + "mendoza": 23680, + "meng": 37102, + "meng": 37450, + "mening": 46428, + "menon": 38255, + "menopau": 34974, + "menopause": 46026, + "mens": 16924, + "mens": 10495, + "mensfashion": 27578, + "menstru": 28345, + "menstrual": 40915, + "menswear": 18803, + "ment": 1585, + "ment": 777, + "mental": 8611, + "mental": 3448, + "mentalhealth": 20593, + "mentalhealth": 13022, + "mentality": 26647, + "mentally": 14307, + "mentary": 4468, + "mentation": 9512, + "mentday": 40397, + "mente": 40302, + "mente": 36396, + "mented": 9249, + "menting": 14471, + "mention": 43881, + "mention": 6762, + "mentioned": 11948, + "mentioning": 34290, + "mentions": 12334, + "mento": 30582, + "mentor": 45342, + "mentor": 11642, + "mentoring": 19610, + "mentors": 20945, + "mentorship": 33878, + "ments": 1827, + "menu": 6225, + "menus": 33534, + "meo": 30792, + "meow": 39965, + "meow": 17246, + "mep": 27095, + "mer": 1316, + "mer": 2452, + "mera": 20028, + "merc": 34357, + "merc": 44399, + "mercado": 45479, + "merce": 8409, + "mercede": 34959, + "mercedes": 26403, + "mercedes": 10685, + "mercedesam": 40107, + "mercedesbenz": 32347, + "mercen": 40301, + "mercer": 21632, + "merch": 11504, + "merchandi": 14954, + "merchandise": 16808, + "merchandising": 49196, + "merchant": 19563, + "merchants": 34427, + "merci": 23364, + "merci": 29378, + "mercur": 11471, + "mercury": 45203, + "mercury": 12653, + "mercy": 33249, + "mercy": 10815, + "mere": 29657, + "mere": 10342, + "mered": 24657, + "mered": 32297, + "meredith": 25103, + "merely": 28718, + "merge": 30406, + "merged": 46492, + "merger": 24744, + "merging": 49256, + "meri": 17993, + "meri": 36109, + "meria": 48433, + "meric": 27097, + "merica": 30561, + "meridi": 37901, + "meridian": 31195, + "mering": 41060, + "meringue": 41661, + "merino": 42648, + "merit": 20830, + "merkel": 24715, + "merle": 48586, + "merlin": 26517, + "merlot": 40424, + "mermaid": 16064, + "mermaids": 43617, + "mero": 19097, + "merr": 48288, + "merri": 21462, + "merrill": 47713, + "merritt": 36462, + "merry": 14167, + "merry": 5779, + "merrychristmas": 19672, + "mers": 4199, + "mersal": 36711, + "mersey": 25248, + "mersey": 46239, + "merseyside": 35382, + "mert": 48496, + "merton": 35315, + "mery": 40873, + "meryl": 35787, + "mes": 28432, + "mes": 3029, + "mesa": 18956, + "mese": 42018, + "mesh": 15030, + "mesm": 18695, + "mesmer": 38435, + "mesmeri": 25985, + "mesmerizing": 35637, + "meso": 25537, + "mesqu": 46819, + "mess": 2490, + "mess": 8188, + "message": 3918, + "messages": 9390, + "messaging": 23234, + "messe": 40391, + "messed": 23580, + "messenger": 17389, + "messi": 19394, + "messi": 11252, + "messiah": 28737, + "messing": 23144, + "messy": 15987, + "mest": 23780, + "mester": 47349, + "mesut": 49177, + "met": 5249, + "met": 2340, + "meta": 14803, + "meta": 22701, + "metab": 16150, + "metabol": 48389, + "metaboli": 25573, + "metabolic": 34311, + "metabolism": 27824, + "metal": 8935, + "metal": 4044, + "metall": 19084, + "metallic": 17257, + "metallica": 24079, + "metals": 21375, + "metam": 28862, + "metamor": 39030, + "metamorpho": 47601, + "metaph": 24189, + "metaphor": 34233, + "metast": 41973, + "mete": 11226, + "meteor": 26429, + "meteor": 26823, + "meteoro": 25948, + "meteorologist": 42849, + "meter": 10104, + "meters": 13247, + "metgala": 30089, + "meth": 21867, + "meth": 26177, + "methane": 37565, + "metho": 5770, + "method": 10284, + "methodist": 25165, + "methodo": 28488, + "methodology": 37316, + "methods": 12200, + "methyl": 48999, + "metmuseum": 28207, + "meto": 25679, + "metoo": 24722, + "metr": 15086, + "metre": 27889, + "metres": 19798, + "metric": 19950, + "metrical": 40704, + "metrics": 24396, + "metro": 7257, + "metro": 6784, + "metroid": 39957, + "metropolis": 40476, + "metropolitan": 19013, + "metry": 20039, + "mets": 9633, + "mett": 28081, + "metz": 40506, + "meu": 34520, + "mew": 40368, + "mex": 3213, + "mex": 18387, + "mexic": 31728, + "mexican": 37442, + "mexican": 8186, + "mexicans": 47729, + "mexico": 31834, + "mexico": 4604, + "mey": 28584, + "mey": 27777, + "meyer": 13963, + "meyers": 32326, + "mez": 30615, + "mez": 46833, + "mezz": 38771, + "mf": 18199, + "mf": 11067, + "mfa": 24107, + "mfc": 39474, + "mfg": 21912, + "mfw": 27309, + "mg": 10003, + "mg": 8014, + "mga": 23954, + "mgm": 27572, + "mgmt": 22288, + "mgr": 31500, + "mgs": 48073, + "mgt": 48663, + "mh": 9962, + "mh": 10834, + "mha": 41944, + "mhealth": 41225, + "mhs": 28815, + "mhz": 31550, + "mi": 714, + "mi": 2251, + "mia": 5852, + "miam": 31053, + "miami": 15106, + "miami": 4891, + "mian": 24792, + "miaw": 36046, + "mib": 48178, + "mic": 1213, + "mic": 3816, + "mica": 41551, + "micah": 33870, + "mice": 19030, + "mich": 25628, + "mich": 23029, + "micha": 2083, + "michael": 6051, + "michael": 2511, + "michaela": 41897, + "michaeljackson": 33532, + "michaels": 23868, + "michal": 47144, + "miche": 37966, + "micheal": 43709, + "michel": 5158, + "michel": 17153, + "michelangelo": 41245, + "michele": 20642, + "michelin": 26330, + "michelle": 19028, + "michelle": 8625, + "michi": 5658, + "michigan": 32344, + "michigan": 6296, + "mick": 15171, + "mick": 12592, + "mickey": 41813, + "mickey": 13053, + "micky": 43011, + "micro": 3160, + "micro": 11374, + "microbes": 44671, + "microbi": 19496, + "microbial": 30335, + "microbiology": 35348, + "microbiome": 35148, + "micron": 48742, + "microphone": 24643, + "micropoetry": 35997, + "microscope": 29114, + "microscopy": 38431, + "microsof": 42424, + "microsoft": 38650, + "microsoft": 7254, + "microwave": 24240, + "mics": 16554, + "mid": 2192, + "mid": 4734, + "midcentury": 48988, + "midd": 2983, + "midday": 23390, + "middle": 9849, + "middle": 3694, + "middleeast": 32783, + "middles": 29769, + "middlesbrough": 32436, + "middlesex": 39154, + "middleton": 23627, + "middleweight": 35829, + "midfield": 28116, + "midfielder": 13423, + "midget": 30734, + "midi": 39496, + "midi": 27326, + "midland": 24822, + "midlands": 18062, + "midnight": 35746, + "midnight": 6302, + "mids": 40821, + "midst": 24752, + "midsummer": 35234, + "midterm": 34365, + "midterms": 32015, + "midtown": 26069, + "midway": 26536, + "midweek": 29120, + "midwest": 16627, + "midwi": 44802, + "midwife": 37681, + "midwives": 42355, + "mie": 20865, + "mie": 10555, + "miento": 46482, + "mier": 36490, + "mies": 8840, + "miff": 49398, + "mig": 28743, + "might": 2727, + "mighty": 26632, + "mighty": 7815, + "mign": 41678, + "migos": 44640, + "migr": 3736, + "migra": 28186, + "migraine": 35360, + "migrant": 18902, + "migrants": 15814, + "migrate": 41804, + "migrating": 43604, + "migration": 11891, + "migu": 12279, + "miguel": 33672, + "miguel": 14436, + "miho": 46870, + "mii": 39896, + "mik": 15096, + "mik": 46203, + "mika": 28609, + "mika": 25185, + "mike": 5884, + "mike": 3178, + "mikel": 48865, + "mikequind": 33508, + "mikequindazzi": 33551, + "mikey": 34934, + "mikey": 23368, + "mikha": 30999, + "mikhail": 38327, + "miki": 48863, + "miko": 35413, + "miku": 37703, + "mil": 1469, + "mil": 12826, + "mila": 26183, + "milan": 30380, + "milan": 8552, + "milano": 18585, + "milb": 42248, + "mild": 16085, + "mildly": 49059, + "mile": 7833, + "mile": 6243, + "mileage": 30579, + "miler": 44680, + "miles": 3446, + "milestone": 13485, + "milestones": 34025, + "miley": 25336, + "miley": 14321, + "mileycyrus": 28528, + "milf": 45386, + "milford": 35840, + "mili": 16698, + "miliband": 41440, + "milit": 3715, + "militant": 33629, + "militants": 23974, + "military": 24498, + "military": 4323, + "militi": 46625, + "militia": 32114, + "milk": 13409, + "milk": 5205, + "milkshake": 29066, + "milky": 37320, + "milky": 21120, + "milkyway": 43246, + "mill": 4221, + "mill": 6637, + "milla": 49381, + "millan": 34930, + "millan": 22188, + "millar": 41851, + "mille": 34066, + "millen": 48501, + "millenni": 10406, + "millennial": 28357, + "millennials": 18804, + "millennium": 21116, + "miller": 21699, + "miller": 5733, + "milli": 5340, + "millie": 29283, + "milling": 39133, + "million": 13154, + "million": 2506, + "millionaire": 25179, + "millionaires": 47159, + "millions": 8492, + "mills": 10331, + "millwall": 35902, + "milly": 45794, + "milne": 44590, + "milner": 45230, + "milo": 24548, + "milton": 39004, + "milton": 17360, + "milwau": 13452, + "milwaukee": 14259, + "mim": 39379, + "mimi": 27086, + "mimic": 47116, + "mimic": 46519, + "mimo": 45551, + "min": 771, + "min": 3331, + "mina": 15281, + "minaj": 25136, + "minal": 40222, + "minat": 33275, + "mince": 32396, + "mind": 5890, + "mind": 2575, + "mindanao": 44228, + "minded": 21330, + "mindful": 28457, + "mindfulness": 15707, + "minding": 45337, + "minds": 9244, + "mindset": 14217, + "mindy": 46875, + "mindy": 38551, + "mine": 20149, + "mine": 3347, + "minecraft": 15678, + "mined": 48034, + "minent": 12533, + "miner": 14109, + "miner": 26572, + "mineral": 17692, + "minerals": 21169, + "miners": 22119, + "mines": 16211, + "ming": 10868, + "ming": 2107, + "mingham": 7590, + "mingle": 38437, + "mingly": 36909, + "mington": 49283, + "mington": 23119, + "minh": 48734, + "minho": 21318, + "mini": 1810, + "mini": 3954, + "miniature": 44298, + "miniature": 16377, + "miniatures": 38816, + "minic": 31522, + "minim": 10005, + "minimal": 18458, + "minimalism": 42594, + "minimalist": 26641, + "minimize": 38697, + "minimum": 12244, + "minindia": 28458, + "mining": 8473, + "minion": 28622, + "minions": 27035, + "minis": 33409, + "minis": 35976, + "minister": 25688, + "minister": 3569, + "ministerial": 33008, + "ministers": 16406, + "ministries": 27895, + "ministry": 8742, + "mink": 42017, + "minn": 45991, + "minn": 47318, + "minne": 7083, + "minneapolis": 16977, + "minneso": 9380, + "minnesota": 9968, + "minnie": 24493, + "mino": 22791, + "minogue": 44202, + "minor": 8522, + "minorities": 28119, + "minority": 16210, + "minors": 36789, + "mins": 6196, + "minsk": 46151, + "minster": 11189, + "mint": 48084, + "mint": 7506, + "minted": 49377, + "minton": 20050, + "minu": 29064, + "minus": 15358, + "minute": 28931, + "minute": 4497, + "minutes": 3056, + "mio": 26366, + "mir": 2750, + "mir": 6585, + "mira": 21665, + "mira": 22762, + "mirac": 13685, + "miracle": 49208, + "miracle": 11543, + "miracles": 23478, + "miraculous": 38671, + "mirage": 28679, + "mirai": 49060, + "mirand": 32367, + "miranda": 17590, + "mire": 38140, + "mire": 30140, + "miri": 22273, + "miriam": 30950, + "miro": 34851, + "miro": 48317, + "mirren": 47600, + "mirro": 48500, + "mirror": 29823, + "mirror": 7220, + "mirrors": 21823, + "mirza": 36440, + "mis": 866, + "mis": 11239, + "mischief": 33896, + "misconceptions": 48681, + "misconduct": 30601, + "mise": 46567, + "mise": 17267, + "miser": 33394, + "miserable": 26196, + "misery": 28360, + "mises": 24390, + "misfits": 42708, + "mish": 15494, + "mish": 20981, + "misha": 35434, + "mishra": 33042, + "misleading": 30862, + "mism": 15948, + "miso": 27657, + "miso": 33441, + "misogy": 31315, + "misogyny": 48415, + "miss": 6984, + "miss": 1526, + "missal": 38337, + "missed": 3955, + "misses": 15844, + "missi": 3008, + "missile": 14411, + "missiles": 27868, + "missin": 36209, + "missing": 23509, + "missing": 3423, + "mission": 12738, + "mission": 2406, + "missionaries": 40580, + "missionary": 27915, + "missions": 6990, + "mississ": 26483, + "mississauga": 28393, + "mississi": 11687, + "mississippi": 12232, + "missou": 30710, + "missoula": 48549, + "missouri": 11835, + "missuni": 26347, + "missuniverse": 28766, + "missy": 48105, + "missy": 31515, + "missyou": 45799, + "mist": 12610, + "mist": 11946, + "mistak": 20478, + "mistake": 11303, + "mistaken": 29182, + "mistakenly": 48494, + "mistakes": 12824, + "mister": 26949, + "mister": 18895, + "mistle": 46800, + "mistletoe": 48569, + "mistre": 42039, + "mistress": 24349, + "mists": 28636, + "misty": 18799, + "misunderstood": 41574, + "misuse": 40970, + "mit": 3303, + "mit": 4551, + "mita": 47514, + "mitage": 27964, + "mitch": 6969, + "mitch": 14150, + "mitchell": 39339, + "mitchell": 9007, + "mite": 26929, + "mith": 21752, + "mith": 17948, + "miti": 17857, + "mitigate": 42273, + "mitigation": 35514, + "mito": 38254, + "mitochondri": 42132, + "mitra": 47703, + "mits": 24086, + "mitsu": 17905, + "mitsubi": 21604, + "mitsubishi": 23030, + "mitt": 17321, + "mitt": 21341, + "mitted": 10307, + "mitting": 27938, + "mitz": 41827, + "mium": 35891, + "miwx": 43941, + "mix": 3210, + "mix": 3285, + "mixed": 29376, + "mixed": 6780, + "mixer": 17200, + "mixers": 39175, + "mixes": 19061, + "mixing": 15588, + "mixtape": 11044, + "mixture": 28286, + "miy": 25695, + "miya": 36257, + "miz": 20881, + "miz": 30795, + "mize": 19076, + "mized": 43418, + "mizing": 38715, + "mizz": 19985, + "mizzou": 26165, + "mj": 13117, + "mj": 14733, + "mk": 11581, + "mk": 8937, + "mke": 36642, + "mkt": 24814, + "ml": 3627, + "ml": 5780, + "mla": 16723, + "mlas": 48464, + "mlb": 21039, + "mlb": 7482, + "mley": 40329, + "mlg": 45801, + "mlin": 24556, + "mlk": 17941, + "mlkday": 39905, + "mlm": 37611, + "mln": 18971, + "mlp": 23620, + "mlpfi": 45475, + "mlpfim": 45640, + "mls": 13077, + "mm": 1028, + "mm": 2848, + "mma": 34140, + "mma": 6096, + "mmc": 44253, + "mme": 13105, + "mmed": 19570, + "mmer": 35717, + "mmer": 7508, + "mmers": 28128, + "mmes": 42862, + "mmi": 34147, + "mming": 21038, + "mming": 16507, + "mmings": 31357, + "mmit": 41050, + "mmj": 43015, + "mmm": 37908, + "mmm": 7641, + "mmmm": 36312, + "mmmm": 13180, + "mmmmm": 21808, + "mmmmmm": 43740, + "mmo": 30418, + "mmon": 41131, + "mmor": 36657, + "mmorpg": 39476, + "mms": 37803, + "mmva": 42666, + "mmy": 28837, + "mmy": 8722, + "mn": 5086, + "mn": 4057, + "mna": 34877, + "mnd": 44776, + "mnet": 34129, + "mnf": 41105, + "mnl": 32980, + "mnleg": 42653, + "mns": 39040, + "mnt": 21477, + "mntwins": 45448, + "mnwild": 39044, + "mnwx": 39592, + "mo": 617, + "mo": 2080, + "moa": 33174, + "moana": 43241, + "mob": 2818, + "mob": 12754, + "mobi": 9451, + "mobil": 26343, + "mobil": 29815, + "mobile": 12935, + "mobile": 3451, + "mobiles": 44302, + "mobili": 20770, + "mobility": 12546, + "mobilization": 48916, + "moby": 47219, + "moc": 41439, + "moc": 36992, + "mocha": 28425, + "mochi": 47973, + "mock": 15641, + "mock": 12759, + "mocked": 47400, + "mocking": 28692, + "mocking": 37870, + "mocks": 35142, + "mod": 6362, + "mod": 10893, + "moda": 25814, + "modal": 33157, + "mode": 20402, + "mode": 6493, + "model": 4591, + "model": 2863, + "modeled": 39527, + "modeling": 13706, + "modelling": 19946, + "models": 6176, + "moder": 2894, + "moderate": 16435, + "moderated": 27928, + "moderating": 34242, + "moderator": 32659, + "modern": 11706, + "modern": 4077, + "modernart": 34417, + "moderni": 24328, + "modernism": 39601, + "modernist": 36773, + "modernization": 47294, + "modes": 30454, + "modest": 25436, + "modi": 9047, + "modi": 7774, + "modification": 37630, + "modified": 17964, + "modo": 36820, + "mods": 23843, + "modu": 9036, + "modular": 22437, + "module": 16757, + "modules": 30575, + "moe": 38655, + "moe": 17938, + "mof": 30798, + "moff": 27160, + "mog": 42362, + "moga": 41732, + "mogadishu": 45133, + "mogul": 41320, + "moh": 18979, + "moh": 35388, + "moha": 46892, + "moham": 7923, + "mohamed": 18472, + "mohammad": 19926, + "mohammed": 16168, + "mohan": 26521, + "mohan": 23586, + "mohawk": 34942, + "mohd": 49094, + "mohsin": 48861, + "moi": 20691, + "moi": 21825, + "moil": 30349, + "moines": 32091, + "moist": 19831, + "moist": 33263, + "moisture": 20412, + "moisturi": 25942, + "moj": 34505, + "moja": 49055, + "mojito": 46830, + "mojo": 25204, + "mok": 49146, + "mol": 4246, + "mol": 31582, + "mold": 21846, + "molding": 46274, + "moldova": 47317, + "mole": 9927, + "mole": 23529, + "molecular": 19370, + "molecule": 39233, + "molecules": 35643, + "molina": 34201, + "mollie": 48203, + "molly": 24368, + "molly": 12573, + "molo": 41510, + "mology": 32255, + "molten": 46071, + "moly": 47083, + "mom": 1614, + "mom": 2543, + "moma": 33605, + "mombasa": 40340, + "moment": 12197, + "moment": 2495, + "momento": 30078, + "moments": 5251, + "momentum": 15722, + "momlife": 43825, + "momma": 14508, + "mommy": 12456, + "momo": 48490, + "momo": 25980, + "moms": 28446, + "moms": 10042, + "momsdemand": 33744, + "mon": 749, + "mon": 2173, + "mona": 19143, + "monaco": 14938, + "monaghan": 39797, + "monarch": 27235, + "monarch": 22619, + "monarchs": 36750, + "monarchy": 47503, + "monaster": 19422, + "monastery": 21850, + "monc": 34847, + "moncton": 44962, + "mond": 14522, + "mond": 4475, + "monday": 6205, + "monday": 2098, + "mondaymorning": 40089, + "mondaymotiv": 45488, + "mondaymotivation": 8198, + "mondaymotivaton": 47034, + "mondays": 13815, + "monde": 29339, + "mondo": 36207, + "monds": 20317, + "mone": 25990, + "monet": 24499, + "monetary": 26394, + "moneti": 38056, + "money": 12743, + "money": 2327, + "mong": 43566, + "monger": 38928, + "mongers": 27670, + "mongo": 20680, + "mongolia": 27144, + "mongolian": 46335, + "moni": 46851, + "monia": 31161, + "monic": 30893, + "monica": 13540, + "monit": 9014, + "monitor": 10198, + "monitored": 45828, + "monitoring": 11030, + "monitors": 30478, + "monk": 30557, + "monk": 16424, + "monkey": 29597, + "monkey": 9465, + "monkeys": 15781, + "monks": 29090, + "monmouth": 36929, + "mono": 8220, + "mono": 22537, + "monochrome": 25576, + "monogram": 39665, + "monologue": 47776, + "monopoly": 25241, + "monoxide": 49314, + "monro": 45750, + "monroe": 13625, + "mons": 19885, + "monsanto": 37592, + "monsi": 46677, + "monsieur": 48879, + "monsoon": 18872, + "monsta": 30718, + "monstax": 45631, + "monste": 47045, + "monster": 14454, + "monster": 6060, + "monsters": 11546, + "mont": 5186, + "mont": 5382, + "montag": 37202, + "montage": 32325, + "montal": 42126, + "montan": 28405, + "montana": 11436, + "monte": 8711, + "monte": 14667, + "montene": 28538, + "montenegro": 30378, + "monter": 36673, + "monterey": 23388, + "monterrey": 45254, + "montess": 43205, + "montessori": 45443, + "montgom": 13852, + "montgomery": 14951, + "month": 7680, + "month": 1924, + "monthly": 8764, + "months": 3109, + "monthsary": 42420, + "monton": 41961, + "montp": 39523, + "montre": 8434, + "montreal": 9262, + "montrose": 42347, + "monty": 43997, + "monty": 24038, + "monu": 9748, + "monument": 12019, + "monumental": 31297, + "monuments": 26916, + "mony": 4117, + "monza": 40380, + "moo": 4953, + "moo": 24626, + "mood": 42358, + "mood": 5394, + "moods": 43727, + "moody": 17170, + "moom": 36887, + "moon": 6334, + "moon": 3293, + "mooney": 37942, + "moonlight": 20001, + "moons": 29887, + "moonshine": 46706, + "moor": 14817, + "moor": 11877, + "moore": 28613, + "moore": 6708, + "moors": 32577, + "moose": 37562, + "moose": 17338, + "moot": 46895, + "mop": 33900, + "mopar": 41166, + "mor": 657, + "mor": 18614, + "mora": 29262, + "moral": 11246, + "morale": 39404, + "morales": 27117, + "morality": 34133, + "morally": 42519, + "morals": 46223, + "moran": 21557, + "moray": 44569, + "more": 5434, + "more": 750, + "morecam": 37305, + "morecambe": 43414, + "mored": 20195, + "moreland": 44135, + "moreno": 24826, + "morethan": 30889, + "morg": 34284, + "morgan": 15432, + "morgan": 6075, + "morgen": 35106, + "mori": 25710, + "mori": 29514, + "moris": 43131, + "moritz": 45594, + "morley": 40439, + "mormon": 27715, + "morn": 22393, + "mornin": 28327, + "morning": 10769, + "morning": 1119, + "mornings": 12106, + "moro": 31613, + "moroc": 11996, + "moroccan": 27546, + "morocco": 15228, + "moron": 31875, + "morons": 46477, + "morow": 40779, + "morph": 23915, + "morph": 41700, + "morphe": 38978, + "morpho": 38622, + "morrha": 43044, + "morri": 9876, + "morris": 22560, + "morris": 9090, + "morrison": 40961, + "morrison": 14094, + "morrisons": 40965, + "morrissey": 30040, + "morro": 48363, + "morrow": 21611, + "mors": 13064, + "morse": 25282, + "mort": 24257, + "mort": 30583, + "mortal": 31883, + "mortal": 14680, + "mortality": 20347, + "mortar": 27258, + "mortg": 12069, + "mortgage": 13988, + "mortgages": 45391, + "mortimer": 47836, + "morton": 20698, + "morty": 37391, + "mory": 22633, + "mos": 28658, + "mos": 9593, + "mosa": 14164, + "mosa": 23809, + "mosaic": 17506, + "mosch": 47003, + "mosco": 9840, + "moscow": 10371, + "moseley": 47080, + "moses": 18451, + "mosley": 46228, + "mosqu": 15215, + "mosque": 12694, + "mosques": 41214, + "mosquit": 39699, + "mosquito": 25083, + "mosquitoes": 41870, + "moss": 25107, + "moss": 12815, + "most": 7034, + "most": 1096, + "mostly": 8829, + "mosul": 29165, + "mot": 16352, + "mot": 15452, + "mota": 42499, + "motd": 46232, + "motel": 26191, + "moth": 33208, + "moth": 11736, + "mother": 7455, + "mother": 3050, + "motherhood": 32274, + "motherland": 46774, + "mothers": 10546, + "mothersday": 15583, + "motherwell": 48104, + "moths": 29086, + "moti": 38210, + "motif": 35373, + "motion": 32139, + "motion": 7860, + "motiv": 3183, + "motivate": 26771, + "motivated": 16521, + "motivates": 44684, + "motivating": 37720, + "motivation": 26117, + "motivation": 4193, + "motivational": 32832, + "motivational": 20472, + "motivationmonday": 28703, + "motive": 36669, + "motley": 42553, + "motm": 41192, + "moto": 10646, + "moto": 11431, + "motocross": 34562, + "motogp": 16615, + "motor": 3975, + "motor": 7659, + "motorbike": 33341, + "motorcycle": 10297, + "motorcycles": 24869, + "motoring": 44491, + "motorists": 32766, + "motorola": 33738, + "motors": 14989, + "motorsport": 18371, + "motorsports": 24264, + "motorway": 31808, + "motown": 32685, + "mott": 44570, + "mott": 21708, + "motto": 23338, + "mou": 2809, + "mou": 25289, + "moud": 37698, + "moul": 25725, + "mould": 36743, + "moulin": 47656, + "moun": 2023, + "mound": 21414, + "mount": 20553, + "mount": 5532, + "mountain": 14547, + "mountain": 3965, + "mountaine": 24841, + "mountaineer": 49255, + "mountains": 5873, + "mounted": 17897, + "mounting": 29910, + "mounts": 36767, + "mour": 9053, + "mour": 42446, + "moured": 29555, + "mourinho": 18536, + "mourn": 33592, + "mourning": 24169, + "mourns": 42811, + "mous": 24837, + "mous": 17425, + "mouse": 33032, + "mouse": 9301, + "mousse": 31869, + "moustache": 32795, + "mouth": 15152, + "mouth": 4932, + "mouths": 38518, + "mov": 23950, + "move": 16624, + "move": 2783, + "moved": 6997, + "movember": 23474, + "movement": 5208, + "movements": 19665, + "mover": 37673, + "movers": 33957, + "moves": 6880, + "movi": 1707, + "movic": 43838, + "movie": 11247, + "movie": 2016, + "movies": 4772, + "moving": 32160, + "moving": 3584, + "mow": 31006, + "mow": 36329, + "mower": 30895, + "mowing": 46424, + "mowx": 44263, + "moy": 27276, + "moy": 34205, + "moyes": 37119, + "moz": 14761, + "moz": 43738, + "mozam": 26648, + "mozambique": 28831, + "mozart": 22132, + "mozz": 26317, + "mozzarella": 27845, + "mp": 1037, + "mp": 1246, + "mpa": 30749, + "mpc": 38560, + "mpd": 33814, + "mped": 28134, + "mper": 22803, + "mpg": 39830, + "mpg": 37454, + "mpgvip": 42149, + "mph": 5306, + "mpi": 43263, + "mping": 27999, + "mple": 21139, + "mplo": 47071, + "mpls": 34298, + "mpo": 33674, + "mpp": 39570, + "mps": 5504, + "mption": 9717, + "mpton": 27448, + "mpu": 47156, + "mpus": 25864, + "mpy": 17192, + "mq": 19103, + "mqm": 24687, + "mr": 3139, + "mr": 1982, + "mra": 44568, + "mrc": 25897, + "mri": 24773, + "mrs": 25003, + "mrs": 4255, + "mrt": 30256, + "mru": 22370, + "mrw": 15303, + "ms": 3525, + "ms": 988, + "msa": 36306, + "msc": 31826, + "msc": 20529, + "msd": 25804, + "msd": 36407, + "msdhoni": 32850, + "msf": 36239, + "msg": 44430, + "msg": 10928, + "msh": 41751, + "msi": 43597, + "msi": 45278, + "msk": 38501, + "msl": 42736, + "msm": 22210, + "msn": 18824, + "msn": 41042, + "msnbc": 20245, + "mson": 27773, + "mson": 12298, + "msp": 41445, + "msp": 22318, + "mss": 42136, + "mss": 48610, + "mst": 26335, + "msu": 26763, + "msu": 17298, + "mswx": 42957, + "msy": 43919, + "mt": 4252, + "mt": 3284, + "mta": 28691, + "mtb": 48306, + "mtb": 18747, + "mtc": 42482, + "mtg": 49142, + "mtg": 13648, + "mth": 48151, + "mtl": 22135, + "mtn": 26041, + "mtn": 18953, + "mtr": 46650, + "mts": 38751, + "mtv": 8099, + "mtv": 12555, + "mtvbr": 47258, + "mtvhottest": 16751, + "mtvstars": 19948, + "mu": 670, + "mu": 6411, + "mua": 21395, + "muay": 44910, + "muaythai": 47763, + "mubarak": 17957, + "muc": 49115, + "much": 14300, + "much": 1238, + "mucha": 42191, + "muchas": 26278, + "mucho": 19864, + "muck": 44731, + "muck": 45330, + "mud": 17491, + "mud": 11673, + "mudder": 49104, + "muddy": 21524, + "mue": 44383, + "mue": 40717, + "mueller": 46863, + "mueller": 14719, + "muen": 48646, + "muer": 33840, + "muf": 33852, + "mufc": 9013, + "muffin": 22696, + "muffins": 25922, + "mufti": 44930, + "mug": 16339, + "mug": 9722, + "mugabe": 36441, + "mughal": 37508, + "mugs": 22852, + "mugshot": 40028, + "muh": 36335, + "muh": 46475, + "muham": 10043, + "muhammad": 12259, + "muir": 44650, + "muir": 24745, + "muj": 44635, + "muk": 17327, + "muk": 32600, + "mukher": 34575, + "mukherjee": 37862, + "mul": 1899, + "mul": 43193, + "mula": 40937, + "mulator": 17463, + "mulberry": 39221, + "mule": 28695, + "mull": 17313, + "mull": 35310, + "mulled": 44641, + "mullen": 30797, + "muller": 33956, + "mullet": 35010, + "mulligan": 44336, + "mullins": 41265, + "mult": 34219, + "multi": 3947, + "multi": 6400, + "multic": 21683, + "multicul": 28004, + "multicultural": 34667, + "multil": 27975, + "multimedia": 27977, + "multin": 38996, + "multinational": 46540, + "multip": 40314, + "multiplayer": 27460, + "multiple": 6470, + "multipurpose": 47665, + "multit": 27814, + "multitasking": 48684, + "mulus": 26180, + "mum": 15565, + "mum": 4030, + "mumb": 5850, + "mumbai": 24279, + "mumbai": 6971, + "mumford": 46184, + "mummy": 16301, + "mums": 17868, + "mun": 2617, + "mun": 21059, + "muna": 48424, + "munch": 23587, + "munch": 33299, + "munchies": 44324, + "munchkin": 41305, + "mund": 14244, + "mundo": 20990, + "muni": 27327, + "muni": 39795, + "munich": 13526, + "munici": 12159, + "municipal": 43667, + "municipal": 16600, + "municipality": 29987, + "munition": 32668, + "munro": 36501, + "munster": 27201, + "mup": 21966, + "muppet": 40598, + "muppets": 40187, + "mups": 42195, + "mur": 2144, + "mur": 18293, + "mura": 45176, + "mural": 12315, + "murals": 31499, + "murder": 28136, + "murder": 5787, + "murdered": 13158, + "murderer": 26956, + "murderers": 48472, + "murdering": 36055, + "murders": 22409, + "murdoch": 29037, + "murphy": 48976, + "murphy": 8914, + "murray": 31978, + "murray": 7513, + "murs": 38783, + "mus": 2198, + "mus": 8103, + "musa": 30540, + "musc": 5696, + "muscat": 33322, + "muscle": 27323, + "muscle": 9269, + "muscles": 16786, + "muscular": 30606, + "muse": 2369, + "muse": 15686, + "museo": 36457, + "muses": 48243, + "museu": 27087, + "museum": 15602, + "museum": 2786, + "museums": 15542, + "museumweek": 37996, + "mush": 7635, + "mushroom": 13011, + "mushrooms": 14730, + "musi": 15628, + "music": 4110, + "music": 1179, + "musica": 26668, + "musical": 36002, + "musical": 5173, + "musically": 48893, + "musicals": 36974, + "musichistory": 37890, + "musician": 11179, + "musicians": 12498, + "musicislife": 43311, + "musicmonday": 35887, + "musicvideo": 26764, + "musik": 32986, + "musings": 44961, + "musique": 42250, + "musk": 32143, + "musk": 19063, + "muskete": 32775, + "musketeers": 37993, + "musko": 34987, + "muskoka": 40832, + "musli": 4958, + "muslim": 43795, + "muslim": 7060, + "muslims": 10513, + "muss": 41493, + "mussels": 33393, + "must": 6783, + "must": 2048, + "mustache": 23451, + "mustaf": 23596, + "mustafa": 29000, + "mustang": 42361, + "mustang": 13309, + "mustangs": 22500, + "mustard": 15794, + "muster": 47361, + "mustread": 28978, + "mut": 12598, + "mut": 22839, + "mutant": 28384, + "mutation": 38626, + "mutations": 39651, + "mute": 31252, + "muted": 48028, + "muth": 34280, + "mutil": 39950, + "mutt": 45924, + "mutu": 17574, + "mutual": 15055, + "mutuals": 31158, + "muy": 44625, + "mv": 10580, + "mv": 8269, + "mvc": 40549, + "mvp": 8905, + "mw": 16725, + "mw": 11206, + "mwc": 24289, + "mwf": 48565, + "mx": 21947, + "mx": 9575, + "my": 1152, + "my": 607, + "mya": 31401, + "myal": 42735, + "myan": 13761, + "myanmar": 14764, + "myart": 38826, + "myco": 48362, + "mydayin": 41896, + "mydayinla": 42801, + "mydubai": 43475, + "mye": 27551, + "myel": 40084, + "myers": 15993, + "myjaps": 47939, + "myle": 43700, + "myles": 25511, + "mylife": 30537, + "mylittle": 37757, + "mylittlepony": 45107, + "myo": 16206, + "myr": 20272, + "myra": 35694, + "myri": 34972, + "myrt": 47785, + "myrtle": 27768, + "mys": 11724, + "myself": 3245, + "mysore": 44924, + "myspace": 41382, + "myster": 4669, + "mysteries": 20605, + "mysterious": 12650, + "mystery": 39828, + "mystery": 6711, + "mysti": 28711, + "mystic": 36264, + "mystic": 23722, + "mystical": 34122, + "myth": 20322, + "myth": 13878, + "mythical": 34377, + "mytho": 43857, + "mythology": 22496, + "myths": 18675, + "mz": 29509, + "mz": 33400, + "mzan": 36322, + "mzansi": 43301, + "má": 36842, + "mé": 21890, + "méxico": 46159, + "mü": 28142, + "mün": 41235, + "n": 77, + "n": 333, + "na": 1097, + "na": 1272, + "naa": 37738, + "naacp": 32176, + "nab": 6951, + "nab": 19440, + "nabe": 35111, + "naby": 24800, + "nac": 14557, + "nac": 18950, + "nach": 12168, + "nach": 43622, + "nacho": 35647, + "nachos": 32847, + "nacht": 37261, + "nacional": 38782, + "nad": 6204, + "nad": 43928, + "nada": 31683, + "nadal": 20814, + "nade": 24908, + "nadi": 30512, + "nadia": 27487, + "nadine": 23356, + "nadu": 20936, + "nae": 19374, + "naf": 16161, + "naf": 45956, + "nafta": 43123, + "nag": 6694, + "nag": 23902, + "naga": 45953, + "naga": 38997, + "nagar": 17490, + "nage": 41219, + "nago": 38349, + "nagoya": 43303, + "nagpur": 43328, + "nah": 26421, + "nah": 11129, + "nahi": 35244, + "nai": 6230, + "nai": 10692, + "naia": 31340, + "naidu": 42429, + "naija": 16326, + "naik": 34424, + "nail": 19459, + "nail": 9059, + "nailart": 43532, + "nailed": 19035, + "nails": 8469, + "nair": 27107, + "naira": 39450, + "naire": 48892, + "nairobi": 17756, + "nais": 46396, + "naissance": 44761, + "naive": 43362, + "naj": 30985, + "naji": 32589, + "nak": 9248, + "nak": 25550, + "naked": 46371, + "naked": 11478, + "naku": 39864, + "nal": 14132, + "nal": 3119, + "nale": 27198, + "nall": 32869, + "nally": 26158, + "nam": 1410, + "nam": 12344, + "nama": 39586, + "naman": 27635, + "namaste": 35549, + "name": 18160, + "name": 1981, + "named": 3194, + "nameis": 40831, + "nament": 3916, + "naments": 16540, + "names": 6130, + "namesake": 41298, + "nami": 20393, + "namibia": 23731, + "naming": 19367, + "namjoon": 31986, + "namm": 35524, + "namo": 46013, + "namo": 24854, + "nan": 4375, + "nan": 7750, + "nana": 18761, + "nanaimo": 40518, + "nancy": 21511, + "nancy": 11425, + "nand": 20435, + "nandez": 12764, + "nando": 46044, + "nang": 48148, + "nani": 27980, + "nanny": 31104, + "nano": 15835, + "nano": 22006, + "nanop": 34177, + "nanotechnology": 42235, + "nanow": 46734, + "nant": 22526, + "nantes": 47533, + "nantucket": 41573, + "nao": 39319, + "naom": 34955, + "naomi": 20173, + "nap": 6568, + "nap": 11012, + "napa": 20545, + "napier": 40875, + "napkin": 38930, + "naples": 23560, + "napo": 18715, + "napol": 20122, + "napoleon": 24969, + "napoli": 22445, + "napp": 11359, + "napping": 37657, + "naps": 31317, + "naq": 46453, + "nar": 2977, + "nar": 20145, + "nara": 33823, + "narcis": 25229, + "narcissi": 35442, + "narco": 38461, + "nard": 18216, + "nare": 34853, + "naren": 8468, + "narendr": 9807, + "narendra": 25848, + "narendramodi": 9853, + "narnia": 48693, + "narr": 11845, + "narrated": 43609, + "narrative": 15933, + "narratives": 35117, + "narrator": 46529, + "narrow": 24006, + "narrow": 16652, + "narrowly": 29747, + "naruto": 22732, + "nas": 3090, + "nas": 15250, + "nasa": 6841, + "nasal": 42853, + "nascar": 25723, + "nascar": 7868, + "nasdaq": 26629, + "nash": 6771, + "nash": 13620, + "nasheed": 49176, + "nashgrier": 33372, + "nashville": 45356, + "nashville": 8585, + "nasi": 47987, + "nasir": 47509, + "nassau": 34048, + "nasser": 43559, + "nasty": 32930, + "nasty": 8709, + "nat": 1276, + "nat": 11310, + "nata": 39392, + "natal": 28516, + "natali": 20296, + "natalia": 32978, + "natalie": 36634, + "natalie": 13595, + "natash": 48701, + "natasha": 23093, + "nate": 26643, + "nate": 7587, + "natgeo": 33009, + "natgeo": 25046, + "nath": 22203, + "nath": 19843, + "nathan": 13028, + "nathan": 9711, + "nathanfillion": 47422, + "nathaniel": 32667, + "nati": 1060, + "nati": 13384, + "natic": 44944, + "natin": 44358, + "nation": 2317, + "nation": 2670, + "national": 3126, + "national": 1362, + "nationalbestfriend": 42222, + "nationaldogday": 32227, + "nationalism": 29867, + "nationalist": 25058, + "nationality": 44451, + "nationally": 15130, + "nationalpark": 33060, + "nationalparks": 41204, + "nationals": 10784, + "nationaltrust": 34051, + "nations": 7654, + "nationwide": 13795, + "native": 20639, + "native": 4562, + "natives": 36060, + "nativity": 33988, + "natl": 39225, + "natl": 34465, + "nato": 13139, + "nats": 21106, + "natu": 2775, + "natur": 6800, + "natural": 13198, + "natural": 3288, + "naturally": 12995, + "naturals": 44686, + "nature": 9382, + "nature": 2625, + "naturelovers": 41514, + "naturephotography": 22533, + "natures": 15616, + "natureuk": 46193, + "nau": 5955, + "nau": 32878, + "naught": 41001, + "naughty": 47255, + "naughty": 15101, + "nautical": 31660, + "nav": 3413, + "nav": 25308, + "navajo": 35523, + "naval": 44725, + "naval": 13273, + "navar": 24848, + "navarro": 37104, + "nave": 42704, + "naveen": 43837, + "naver": 32534, + "navi": 16159, + "navi": 44848, + "navig": 12507, + "navigate": 24400, + "navigating": 33134, + "navigation": 20148, + "navigator": 38910, + "navis": 36377, + "navratri": 45428, + "navy": 28414, + "navy": 5598, + "naw": 16259, + "naw": 30500, + "nawaz": 49161, + "nawaz": 19523, + "nax": 38299, + "nay": 11704, + "nay": 16182, + "naya": 38917, + "nayanth": 38157, + "nayanthara": 45184, + "naz": 6363, + "naz": 35534, + "nazi": 12972, + "nazis": 21778, + "nb": 6459, + "nb": 6813, + "nba": 22524, + "nba": 5139, + "nbad": 43458, + "nbaf": 30127, + "nbafinals": 33803, + "nbap": 41956, + "nbaplayoffs": 43860, + "nbat": 46291, + "nbc": 9352, + "nbc": 8799, + "nbd": 24526, + "nbl": 42652, + "nc": 5021, + "nc": 4911, + "nca": 6921, + "ncaa": 9418, + "ncbd": 47221, + "ncc": 33195, + "ncc": 36686, + "ncds": 47573, + "ncfc": 31274, + "ncis": 33617, + "ncpol": 40562, + "ncr": 38474, + "ncs": 42689, + "nct": 27723, + "nct": 20319, + "ncwx": 36166, + "nd": 5625, + "nd": 1764, + "nda": 32862, + "ndc": 47564, + "ndi": 48229, + "ndp": 19257, + "nds": 31347, + "ndtv": 26261, + "ne": 557, + "ne": 1422, + "nea": 24068, + "neal": 33652, + "neal": 16730, + "near": 11296, + "near": 2252, + "nearby": 13314, + "nearest": 18985, + "nearing": 26571, + "nearly": 4816, + "nears": 37710, + "neat": 43201, + "neat": 15465, + "neath": 18315, + "neau": 31559, + "neb": 40209, + "nebra": 13371, + "nebraska": 14565, + "nebu": 49295, + "nebula": 22532, + "nec": 25109, + "nec": 22992, + "necess": 6961, + "necessarily": 25853, + "necessary": 8955, + "necessities": 43483, + "necessity": 33163, + "neck": 6066, + "neck": 6906, + "necklace": 7385, + "necklaces": 32276, + "necks": 29701, + "nectar": 33683, + "ned": 16030, + "ned": 1369, + "nederland": 49058, + "nee": 20494, + "nee": 10601, + "need": 3229, + "need": 1262, + "needed": 4049, + "needing": 22894, + "needle": 44490, + "needle": 19886, + "needles": 27250, + "needless": 39984, + "needs": 2536, + "needy": 30150, + "neel": 33092, + "neel": 46043, + "neer": 34245, + "nees": 47248, + "neet": 46362, + "neg": 5513, + "negan": 42623, + "negative": 8869, + "negatively": 40254, + "negativity": 34658, + "neglec": 18827, + "neglect": 33680, + "neglected": 31893, + "negli": 32594, + "negligence": 45658, + "negoti": 10216, + "negotiate": 32969, + "negotiating": 35510, + "negotiation": 36504, + "negotiations": 20433, + "negr": 42190, + "negro": 26554, + "neh": 40416, + "neh": 41697, + "neha": 44463, + "nehru": 30316, + "nei": 9366, + "neigh": 4061, + "neighb": 6534, + "neighbor": 7759, + "neighbor": 14485, + "neighborhood": 9471, + "neighborhoods": 26713, + "neighboring": 44754, + "neighbors": 13037, + "neighbour": 15858, + "neighbour": 23719, + "neighbourhood": 20312, + "neighbours": 17594, + "neil": 13591, + "neil": 8030, + "neilhimself": 45682, + "neill": 19324, + "neither": 14398, + "nek": 47727, + "neko": 47066, + "nel": 5476, + "nel": 2693, + "nell": 27081, + "nell": 8117, + "nelly": 21166, + "nels": 19296, + "nelson": 24774, + "nelson": 8586, + "nem": 45153, + "neman": 48553, + "neme": 30993, + "nemesis": 37811, + "nemo": 30441, + "nen": 17817, + "nen": 15451, + "nene": 44167, + "neo": 14562, + "neo": 11017, + "neon": 21043, + "neon": 13919, + "neonatal": 46464, + "neop": 49069, + "nep": 20739, + "nep": 41960, + "nepal": 25597, + "nepal": 10066, + "nepali": 47579, + "neph": 27926, + "nephe": 41810, + "nephew": 11689, + "nephews": 43747, + "nephro": 43054, + "neptune": 30566, + "ner": 2064, + "ner": 998, + "nerd": 24452, + "nerd": 12273, + "nerds": 22609, + "nerdy": 33124, + "nered": 17583, + "nerf": 42914, + "nering": 20226, + "nero": 29048, + "ners": 2129, + "nerve": 18571, + "nerves": 27813, + "nervous": 13928, + "nery": 48597, + "nes": 5457, + "nes": 4980, + "nesburg": 27159, + "nese": 32220, + "ness": 7187, + "ness": 1294, + "nesses": 20107, + "nessy": 32939, + "nest": 20302, + "nest": 8719, + "nesting": 28860, + "nestle": 43967, + "nestled": 38107, + "nests": 41133, + "net": 1851, + "net": 2315, + "netany": 23137, + "netanyahu": 23583, + "netball": 19761, + "netes": 44335, + "netfli": 6304, + "netflix": 35325, + "netflix": 6600, + "nether": 9946, + "netherlands": 11060, + "neti": 43980, + "netneutrality": 47794, + "nets": 8582, + "nett": 23403, + "nett": 6975, + "nette": 13271, + "network": 23285, + "network": 3304, + "networking": 9818, + "networks": 10004, + "neu": 3855, + "neu": 43342, + "neue": 45764, + "neur": 19001, + "neur": 31976, + "neural": 26388, + "neuro": 7401, + "neuro": 36000, + "neurological": 41718, + "neurology": 43197, + "neurons": 40442, + "neuroscience": 23381, + "neutr": 17207, + "neutral": 17011, + "neutrality": 26511, + "neutron": 44056, + "nev": 10236, + "nev": 43645, + "neva": 43304, + "nevada": 13499, + "neve": 44099, + "neve": 44023, + "never": 6746, + "never": 1426, + "neveragain": 45053, + "neverforget": 19242, + "nevergiveup": 42497, + "neverland": 41483, + "nevertheless": 48355, + "nevertrump": 47494, + "neville": 19269, + "nevis": 43670, + "new": 1218, + "new": 686, + "newark": 20240, + "newbie": 45427, + "newborn": 18320, + "newbury": 34169, + "newcastle": 41955, + "newcastle": 9302, + "newcomer": 30648, + "newcomers": 44037, + "newe": 40068, + "newell": 41436, + "newer": 33099, + "newest": 4990, + "newfound": 25250, + "newfoundland": 28079, + "newh": 18546, + "newin": 31911, + "newjersey": 32621, + "newly": 42186, + "newly": 7056, + "newman": 15815, + "newmarket": 38617, + "newmexico": 35238, + "newmusic": 32510, + "newmusic": 17201, + "newor": 25969, + "neworleans": 31205, + "newport": 42580, + "newport": 14846, + "newprofile": 14633, + "newprofilepic": 14754, + "newrelease": 34793, + "news": 6216, + "news": 1120, + "newsat": 43979, + "newsc": 28656, + "newscast": 45031, + "newsle": 10727, + "newsletter": 11069, + "newsnow": 48650, + "newsp": 7109, + "newspaper": 8786, + "newspapers": 22423, + "newsroom": 23200, + "newt": 37224, + "newton": 33122, + "newton": 12606, + "newtown": 31747, + "newyear": 22161, + "newyear": 12999, + "newyearseve": 37587, + "newyork": 18140, + "newyork": 10454, + "newyorkcity": 30460, + "newyorker": 39732, + "newzealand": 21117, + "nex": 6897, + "nex": 39720, + "next": 12434, + "next": 1131, + "nextgen": 41933, + "nexus": 19053, + "ney": 3857, + "ney": 1438, + "neymar": 21878, + "neys": 12616, + "nez": 27388, + "nf": 15195, + "nf": 25643, + "nfamily": 20098, + "nfc": 23695, + "nffc": 27893, + "nfl": 11219, + "nfl": 4691, + "nfldraft": 25002, + "ng": 10352, + "ng": 5215, + "nga": 35477, + "ngc": 29046, + "ngo": 38740, + "ngo": 24821, + "ngos": 34627, + "nguyen": 29947, + "nh": 3760, + "nh": 10803, + "nhc": 44817, + "nhl": 12290, + "nhl": 8167, + "nhlbruins": 39081, + "nhljets": 49357, + "nhm": 39483, + "nhpolitics": 36125, + "nhq": 42368, + "nhra": 30052, + "nhs": 23282, + "nhs": 7695, + "ni": 697, + "ni": 3256, + "nia": 3098, + "niag": 18071, + "niagar": 39298, + "niagara": 18965, + "niall": 41354, + "niall": 8327, + "niallo": 22855, + "niallofficial": 23084, + "niam": 39347, + "nian": 46003, + "nib": 31049, + "nic": 2109, + "nic": 6651, + "nica": 29040, + "nicar": 25119, + "nicaragua": 28423, + "nice": 28386, + "nice": 1805, + "nicely": 12303, + "nicer": 29488, + "nicest": 22967, + "niche": 25279, + "nichol": 7668, + "nicholas": 39814, + "nicholas": 13148, + "nicholls": 38846, + "nichols": 22730, + "nicholson": 28745, + "nick": 4209, + "nick": 4253, + "nickel": 22034, + "nickelo": 28668, + "nickelodeon": 33279, + "nicki": 17738, + "nickimin": 27390, + "nickiminaj": 27593, + "nickjonas": 43862, + "nickname": 24731, + "nicknamed": 45190, + "nicks": 15049, + "nicky": 28893, + "nicky": 22091, + "nico": 20850, + "nico": 17779, + "nicol": 9919, + "nicol": 48274, + "nicola": 21791, + "nicolas": 43813, + "nicolas": 18918, + "nicole": 21246, + "nicole": 10000, + "nicot": 45099, + "nicotine": 46697, + "nie": 9524, + "nie": 3501, + "niece": 12795, + "nieces": 44877, + "niel": 19109, + "niel": 26837, + "niels": 37154, + "nielsen": 28372, + "nier": 13014, + "nies": 10586, + "niest": 15007, + "nieu": 29781, + "nific": 4748, + "nifty": 25604, + "nig": 27933, + "nig": 28099, + "nigan": 48516, + "nigel": 33919, + "nigel": 15153, + "niger": 4524, + "niger": 29920, + "nigeri": 40913, + "nigeria": 6106, + "nigerian": 12167, + "nigerians": 25358, + "nigh": 13525, + "nigh": 48157, + "night": 3870, + "night": 930, + "nightclub": 20418, + "nighter": 41349, + "nighting": 36211, + "nightingale": 40696, + "nightlife": 28823, + "nightly": 28868, + "nightmare": 12867, + "nightmares": 24032, + "nightout": 44257, + "nights": 4296, + "nighttime": 38147, + "nightw": 39956, + "nih": 25783, + "nik": 5126, + "nik": 13705, + "nike": 16300, + "nike": 5783, + "nikeplus": 43154, + "niki": 36136, + "nikita": 37118, + "nikk": 38596, + "nikki": 23156, + "nikki": 16689, + "niko": 43771, + "nikol": 27430, + "nikola": 42146, + "nikon": 25488, + "nikon": 13849, + "nikov": 43960, + "nil": 16852, + "nil": 35030, + "nile": 24252, + "nim": 30402, + "nim": 42093, + "nima": 42586, + "nin": 5794, + "nin": 14145, + "nina": 13891, + "nine": 16213, + "nine": 7330, + "ninety": 48214, + "ning": 6050, + "ning": 762, + "ningham": 23395, + "ningly": 43537, + "nings": 4588, + "nington": 26214, + "ninj": 23225, + "ninja": 11969, + "ninjas": 42796, + "nino": 25633, + "ninten": 6184, + "nintendo": 13969, + "nintendo": 7886, + "nintendoswitch": 16404, + "ninth": 22770, + "nip": 33889, + "nip": 22333, + "nipp": 24634, + "nipple": 45987, + "nipples": 44774, + "nippon": 47960, + "nips": 49241, + "nir": 15503, + "nir": 40057, + "nireland": 45763, + "niro": 47373, + "nirvana": 28300, + "nis": 5609, + "nis": 3786, + "nish": 19834, + "nish": 13256, + "nished": 24141, + "nishi": 32386, + "nishings": 49247, + "nison": 45700, + "niss": 39043, + "nissan": 37635, + "nissan": 11082, + "nist": 17782, + "nister": 36640, + "nit": 4087, + "nit": 19011, + "nite": 8427, + "niti": 43964, + "niti": 45355, + "nitin": 37529, + "nitro": 30726, + "nitrogen": 30706, + "niture": 7840, + "nity": 12707, + "niu": 48187, + "niv": 47300, + "niversary": 29643, + "nix": 48552, + "nix": 32278, + "nixon": 20671, + "nj": 8343, + "nj": 6672, + "njcaa": 48992, + "njpw": 38992, + "nk": 22708, + "nk": 17456, + "nko": 36353, + "nl": 12057, + "nl": 7655, + "nli": 37502, + "nlp": 35680, + "nlwx": 49260, + "nm": 15956, + "nm": 11370, + "nmd": 43331, + "nme": 40454, + "nmwx": 47967, + "nn": 8947, + "nn": 12925, + "nnn": 26277, + "nnnn": 41420, + "no": 578, + "no": 871, + "noaa": 27557, + "noah": 28806, + "noah": 11519, + "nobel": 33742, + "nobel": 15605, + "nobelprize": 46074, + "noble": 29430, + "noble": 12051, + "nobody": 7009, + "noc": 16988, + "noc": 44420, + "nocchi": 46359, + "noch": 38672, + "noche": 29689, + "noches": 44166, + "nock": 16993, + "noctur": 26291, + "nocturnal": 41738, + "nod": 18648, + "nodapl": 39079, + "node": 31434, + "node": 24871, + "nodejs": 39262, + "nodes": 40534, + "noel": 38406, + "noel": 17496, + "nof": 29505, + "noff": 46979, + "nofilter": 16418, + "nog": 31157, + "noh": 40775, + "noi": 43115, + "noi": 39889, + "noida": 33404, + "noir": 39291, + "noir": 12953, + "nois": 22057, + "noise": 41018, + "noise": 9307, + "noises": 31575, + "noisse": 45686, + "noisy": 33495, + "nokia": 17731, + "nol": 8055, + "nola": 13289, + "nolan": 17323, + "nold": 40322, + "nole": 34654, + "noles": 40569, + "nollywood": 43145, + "nology": 42221, + "nom": 2981, + "nom": 12799, + "nomad": 27849, + "noman": 45592, + "nomin": 5643, + "nominate": 17122, + "nominated": 8710, + "nominating": 45747, + "nomination": 14136, + "nominations": 17124, + "nominee": 14122, + "nominees": 17873, + "nomnom": 26962, + "nomore": 35126, + "noms": 35706, + "non": 4282, + "non": 3353, + "none": 29644, + "none": 8906, + "nonetheless": 39675, + "nonfiction": 31654, + "nonprofit": 19315, + "nonprofits": 37935, + "nonsense": 19136, + "nonstop": 30300, + "nont": 25207, + "noo": 6759, + "noo": 46672, + "noodle": 19521, + "noodles": 15782, + "nook": 30088, + "noon": 37693, + "noon": 2347, + "noor": 46978, + "noor": 31323, + "nope": 15625, + "nor": 1062, + "nor": 6190, + "nora": 25890, + "norcal": 41970, + "nord": 19261, + "nord": 36067, + "nordic": 36439, + "nordic": 20734, + "nordstrom": 38562, + "norfolk": 30232, + "norfolk": 12202, + "norm": 10990, + "norm": 22457, + "norma": 35757, + "normal": 28748, + "normal": 5967, + "normali": 45157, + "normally": 15870, + "norman": 22027, + "norman": 11338, + "normandy": 23840, + "normani": 44596, + "norms": 33011, + "norris": 21814, + "norse": 36559, + "norte": 35638, + "north": 3468, + "north": 2188, + "northampton": 49246, + "northampton": 26175, + "northan": 37081, + "northbound": 24228, + "northcarolina": 43386, + "northe": 24675, + "northeast": 42673, + "northeast": 13009, + "northeastern": 28297, + "northeasthour": 42869, + "norther": 26908, + "northern": 17210, + "northern": 5049, + "northernlights": 48940, + "northkorea": 38495, + "northside": 45957, + "northumber": 22295, + "northumberland": 22922, + "northwales": 49371, + "northwest": 12894, + "northwestern": 23685, + "norton": 18032, + "norway": 8780, + "norwe": 14414, + "norwegian": 15971, + "norwich": 37629, + "norwich": 15812, + "norwood": 37889, + "nos": 13420, + "nose": 24192, + "nose": 8231, + "noses": 48163, + "nostal": 12076, + "nostalgia": 16622, + "nostalgic": 24468, + "not": 2534, + "not": 783, + "notable": 22023, + "notch": 19476, + "notdead": 42059, + "note": 10910, + "note": 3246, + "notebook": 16365, + "notebooks": 37623, + "noted": 22501, + "notes": 5795, + "nothin": 24291, + "nothing": 28412, + "nothing": 2586, + "noti": 10686, + "notic": 6915, + "notice": 6683, + "noticeable": 40857, + "noticed": 9324, + "notices": 33459, + "noticias": 47759, + "noticing": 37571, + "notification": 22512, + "notifications": 23169, + "notified": 39454, + "noting": 38649, + "notion": 37856, + "notjust": 33212, + "notjustlakes": 45803, + "notmy": 39301, + "noto": 29878, + "noton": 48258, + "notor": 21711, + "notori": 44065, + "notorious": 22489, + "notre": 24397, + "notre": 15306, + "notredame": 34077, + "notsorry": 34361, + "nott": 9333, + "nott": 34989, + "notte": 47308, + "nottingham": 12852, + "notts": 25598, + "nou": 8751, + "nou": 30953, + "noun": 33663, + "nouri": 23796, + "nourish": 46025, + "nourished": 48354, + "nous": 29485, + "nouveau": 29948, + "nouvel": 34215, + "nov": 2264, + "nov": 4293, + "nova": 11236, + "novak": 26465, + "novasco": 33785, + "novascotia": 34744, + "novation": 39753, + "nove": 30507, + "novel": 15044, + "novel": 6080, + "novelist": 27314, + "novella": 42770, + "novels": 16040, + "novelty": 37750, + "november": 3680, + "nover": 37465, + "novi": 47957, + "novice": 33743, + "novo": 27504, + "novo": 36581, + "now": 2040, + "now": 692, + "nowadays": 26155, + "nowhere": 14108, + "nowplaying": 3708, + "nowwatching": 30852, + "nox": 27406, + "noxi": 39304, + "noxious": 42833, + "noy": 32787, + "np": 18205, + "np": 6314, + "npa": 42378, + "npc": 33966, + "npr": 39941, + "npr": 24078, + "nps": 22025, + "npt": 47231, + "nr": 6574, + "nr": 9713, + "nra": 17286, + "nrc": 45786, + "nrf": 47982, + "nrg": 48662, + "nrl": 27142, + "nrl": 18127, + "ns": 12405, + "ns": 1373, + "nsa": 23004, + "nsc": 32792, + "nsd": 36659, + "nsf": 34180, + "nsfw": 19847, + "nsi": 47824, + "nsw": 21301, + "nsw": 11693, + "nswpol": 44434, + "nt": 10902, + "nt": 3207, + "ntr": 30845, + "nts": 43775, + "ntt": 22859, + "ntv": 24807, + "ntv": 45304, + "nu": 1156, + "nu": 9444, + "nucle": 25693, + "nuclear": 34136, + "nuclear": 7279, + "nude": 16630, + "nudes": 32122, + "nue": 22834, + "nuestra": 45649, + "nuestro": 38590, + "nuev": 47861, + "nueva": 48810, + "nuevo": 30265, + "nufc": 15720, + "nuff": 37324, + "nug": 13471, + "nugent": 47457, + "nugget": 25448, + "nuggets": 18970, + "nuh": 45950, + "nuit": 38815, + "nuk": 39228, + "nuke": 39399, + "nul": 29358, + "null": 47376, + "num": 17896, + "num": 30534, + "numb": 34639, + "numb": 39427, + "number": 44078, + "number": 2842, + "numbered": 25975, + "numbers": 6121, + "numer": 11442, + "numerous": 17082, + "numis": 39100, + "nun": 12511, + "nun": 28540, + "nunavut": 48626, + "nunes": 40697, + "nuns": 44061, + "nup": 46757, + "nur": 3920, + "nur": 33493, + "nure": 42480, + "nurse": 37547, + "nurse": 10058, + "nursery": 15540, + "nurses": 12938, + "nursing": 11126, + "nurture": 38865, + "nurturing": 45229, + "nus": 25157, + "nus": 18239, + "nut": 10358, + "nut": 6491, + "nutcracker": 36733, + "nutella": 27312, + "nutr": 6198, + "nutri": 15470, + "nutrient": 32900, + "nutrients": 24668, + "nutriti": 17978, + "nutrition": 41546, + "nutrition": 7989, + "nutritional": 26457, + "nutritious": 30387, + "nuts": 8644, + "nutshell": 26659, + "nutty": 39846, + "nv": 17217, + "nv": 16985, + "nvi": 22847, + "nvidia": 27325, + "nw": 7826, + "nw": 7030, + "nwa": 34237, + "nwo": 40976, + "nws": 23333, + "nws": 30998, + "nwsl": 48394, + "nwt": 25029, + "nx": 18810, + "nx": 16997, + "nxt": 35037, + "nxt": 17804, + "ny": 1383, + "ny": 1350, + "nya": 24165, + "nyc": 13304, + "nyc": 2832, + "nycc": 27187, + "nycfc": 47497, + "nye": 40723, + "nye": 13416, + "nyfw": 21089, + "nyk": 46841, + "nylon": 25915, + "nyo": 41534, + "nyo": 44586, + "nypd": 42293, + "nypd": 18279, + "nyr": 32538, + "nyrd": 47936, + "nys": 36375, + "nys": 23423, + "nyse": 32650, + "nyt": 46311, + "nyt": 12816, + "nytimes": 13772, + "nyu": 43143, + "nyu": 31355, + "nz": 10142, + "nz": 7082, + "o": 78, + "o": 334, + "oa": 11994, + "oahu": 37790, + "oak": 6010, + "oak": 7221, + "oakland": 42663, + "oakland": 12077, + "oakley": 27810, + "oaks": 16734, + "oakville": 38500, + "oasis": 18185, + "oat": 20095, + "oat": 34132, + "oates": 47094, + "oath": 20108, + "oatmeal": 26374, + "oats": 24150, + "oax": 43090, + "oaxaca": 47818, + "ob": 1411, + "ob": 14908, + "oba": 42902, + "oba": 15147, + "obam": 13174, + "obama": 4276, + "obamacare": 18005, + "obe": 11897, + "obe": 29117, + "obedience": 48921, + "ober": 15284, + "obese": 41757, + "obesity": 19499, + "obey": 26926, + "obi": 21454, + "obi": 18414, + "obile": 20513, + "obitu": 39218, + "obituary": 43580, + "objec": 7970, + "object": 14115, + "objective": 23663, + "objectives": 30238, + "objects": 13770, + "obl": 31452, + "oblast": 42672, + "obli": 11416, + "obligation": 34473, + "obligations": 38232, + "obligatory": 35020, + "oblivion": 45323, + "obo": 46001, + "obo": 26618, + "obrien": 31946, + "obs": 39162, + "obsc": 20392, + "obscure": 33337, + "obse": 8433, + "observ": 9050, + "observation": 20250, + "observations": 27409, + "observatory": 21236, + "observe": 23217, + "observed": 21267, + "observer": 22077, + "observers": 47544, + "observing": 28359, + "obsessed": 9744, + "obsession": 15718, + "obsi": 47323, + "obsole": 35561, + "obsolete": 40628, + "obst": 29398, + "obstac": 24075, + "obstacle": 29751, + "obstacles": 24480, + "obste": 49103, + "obstru": 44876, + "obstruc": 38762, + "obstruction": 40240, + "obtain": 26555, + "obtained": 29322, + "obvious": 13959, + "obviously": 10068, + "oc": 1566, + "oc": 6603, + "oca": 31120, + "ocal": 38148, + "occ": 43940, + "occa": 8530, + "occasion": 12280, + "occasional": 33059, + "occasionally": 32479, + "occasions": 26154, + "occer": 20804, + "occi": 42994, + "occu": 7863, + "occult": 42529, + "occup": 11152, + "occupation": 18624, + "occupational": 30644, + "occupied": 17271, + "occupy": 22453, + "occupy": 24210, + "occur": 11264, + "occur": 21813, + "occurred": 19850, + "occurrence": 40615, + "occurring": 31335, + "occurs": 26563, + "ocd": 35904, + "oce": 3509, + "ocean": 12941, + "ocean": 4918, + "oceans": 16792, + "och": 29334, + "och": 32011, + "oche": 33045, + "oci": 9891, + "ocity": 46039, + "ock": 33579, + "ock": 21313, + "ocks": 22410, + "oclock": 36274, + "oco": 32553, + "ocon": 33090, + "ocr": 45813, + "ocre": 40320, + "ocs": 27297, + "oct": 4565, + "octa": 23444, + "octag": 37768, + "octagon": 49167, + "octane": 43040, + "octavia": 47416, + "octo": 31032, + "october": 3481, + "octopus": 22327, + "ocu": 22709, + "oculus": 30082, + "od": 4886, + "od": 9719, + "oda": 24777, + "oday": 41954, + "odd": 15525, + "odd": 11387, + "oddly": 34213, + "odds": 11555, + "ode": 19125, + "ode": 19639, + "odell": 41556, + "odessa": 43574, + "odi": 12223, + "odi": 18853, + "odin": 35175, + "odisha": 15737, + "odo": 49188, + "odo": 40993, + "odor": 39509, + "odu": 35095, + "odu": 39904, + "odyssey": 19991, + "oe": 24251, + "oe": 11667, + "oec": 24288, + "oecd": 30816, + "oem": 29650, + "oes": 3643, + "of": 684, + "of": 539, + "ofa": 29774, + "ofc": 19877, + "ofe": 30000, + "ofer": 47322, + "off": 892, + "off": 1007, + "offe": 8261, + "offee": 34059, + "offen": 7231, + "offence": 34594, + "offences": 33972, + "offended": 30765, + "offender": 48294, + "offenders": 35878, + "offense": 15253, + "offensive": 11037, + "offer": 20607, + "offer": 3271, + "offered": 9395, + "offering": 6896, + "offerings": 24535, + "offers": 4679, + "offic": 3276, + "office": 18033, + "office": 2171, + "officeof": 38750, + "officeofrg": 47100, + "officer": 4683, + "officers": 6335, + "offices": 10933, + "offici": 1401, + "official": 5768, + "official": 1868, + "officially": 4226, + "officials": 7658, + "officiel": 26548, + "offl": 16851, + "offline": 22724, + "offro": 32198, + "offroad": 37173, + "offs": 23987, + "offseason": 25485, + "offset": 28843, + "offshore": 15496, + "offside": 49347, + "offspring": 38635, + "offthe": 38189, + "ofi": 36692, + "ofi": 49090, + "oficial": 18061, + "oft": 16693, + "oftball": 39768, + "often": 4864, + "ofthe": 7592, + "oftheday": 6988, + "oftheweek": 20654, + "oftheyear": 33975, + "og": 11542, + "og": 8555, + "oga": 47312, + "ogden": 42011, + "ogil": 39013, + "ography": 22399, + "ogue": 24761, + "ogun": 48970, + "oh": 5648, + "oh": 1779, + "ohana": 48330, + "ohh": 23076, + "ohhh": 27697, + "ohhhh": 40201, + "ohi": 5207, + "ohio": 18951, + "ohio": 6155, + "ohiostate": 41324, + "ohl": 45547, + "ohl": 41095, + "ohmy": 29758, + "ohn": 48043, + "ohs": 39542, + "ohwx": 47993, + "oi": 27357, + "oi": 13934, + "oic": 45554, + "oid": 14758, + "oids": 21847, + "oil": 11973, + "oil": 2870, + "oiland": 32316, + "oilandgas": 34130, + "oilers": 21627, + "oilpainting": 34279, + "oils": 17886, + "oily": 47550, + "oir": 48079, + "oir": 37113, + "ois": 23262, + "oit": 18453, + "oitnb": 34865, + "oj": 30986, + "oj": 34553, + "ok": 1944, + "ok": 2481, + "oka": 42258, + "oka": 19092, + "okan": 41263, + "okanagan": 43233, + "okay": 4917, + "okc": 42418, + "okc": 18357, + "oke": 26636, + "oke": 23598, + "oki": 20390, + "okin": 30687, + "okinawa": 35877, + "okla": 9431, + "oklahoma": 10170, + "oko": 26892, + "oko": 26095, + "okstate": 36356, + "oktoberfest": 32026, + "oku": 45010, + "oku": 43829, + "okwx": 27336, + "ol": 562, + "ol": 2985, + "ola": 20499, + "ola": 3373, + "olaf": 39709, + "olan": 48489, + "olan": 24227, + "oland": 26452, + "olas": 40800, + "old": 4931, + "old": 896, + "olde": 37731, + "older": 7700, + "oldest": 9285, + "oldham": 29929, + "oldie": 35280, + "oldies": 36278, + "oldman": 48614, + "olds": 8580, + "oldschool": 44384, + "oldschool": 25133, + "oldsmobile": 45396, + "ole": 9089, + "ole": 1947, + "oled": 46768, + "oler": 24069, + "oles": 16962, + "olf": 16346, + "olga": 34779, + "oli": 3811, + "oli": 8810, + "olic": 31341, + "oligar": 46185, + "olim": 47769, + "olin": 37823, + "olin": 18283, + "olina": 34711, + "oline": 17441, + "oling": 38033, + "olini": 36040, + "olis": 49397, + "olithic": 35574, + "olive": 22486, + "olive": 9898, + "oliver": 22882, + "oliver": 9261, + "olives": 27149, + "olivi": 20773, + "olivia": 11697, + "olivier": 23891, + "oll": 32270, + "oll": 15510, + "olla": 31908, + "ollie": 24434, + "olls": 42697, + "olly": 23998, + "olo": 14628, + "olo": 7606, + "ological": 12345, + "ologist": 23442, + "ologists": 30912, + "ology": 4627, + "olor": 29245, + "olph": 25077, + "ols": 2236, + "olsen": 26307, + "olson": 28046, + "olt": 46252, + "olu": 16502, + "olu": 46302, + "olulu": 27645, + "oly": 20323, + "oly": 24823, + "olym": 3594, + "olympi": 13597, + "olympia": 23965, + "olympiad": 47694, + "olympian": 25420, + "olympians": 44583, + "olympic": 26099, + "olympic": 6388, + "olympics": 7629, + "olympus": 30960, + "om": 547, + "om": 3932, + "oma": 44603, + "oma": 5358, + "omaha": 16509, + "oman": 22088, + "oman": 10871, + "omar": 19488, + "omar": 13367, + "omars": 37099, + "omas": 36023, + "omat": 40788, + "omb": 34447, + "ombe": 35967, + "omd": 49346, + "ome": 3693, + "ome": 5832, + "omed": 16835, + "omega": 13465, + "omelette": 38789, + "omen": 9969, + "omen": 25469, + "oment": 43683, + "omeo": 39844, + "omer": 24087, + "omer": 17902, + "omes": 25736, + "ometer": 20060, + "ometric": 38702, + "omez": 12541, + "omf": 47496, + "omfg": 12523, + "omg": 35233, + "omg": 3186, + "omi": 24097, + "omi": 10341, + "omic": 40536, + "omic": 12793, + "omics": 15138, + "omile": 46915, + "omin": 16457, + "omination": 42571, + "oming": 10796, + "ominous": 40914, + "omni": 18793, + "omni": 39489, + "omnibus": 44760, + "omnic": 48383, + "omo": 14478, + "omo": 11066, + "omon": 48758, + "omor": 29431, + "oms": 3770, + "omusic": 38965, + "omy": 40805, + "omy": 6884, + "on": 521, + "on": 525, + "ona": 2687, + "onair": 29511, + "onal": 918, + "onboard": 21689, + "once": 16331, + "once": 2654, + "onceupon": 28122, + "onceuponatime": 33505, + "onco": 46700, + "oncology": 24593, + "ond": 27918, + "ond": 2636, + "onda": 32643, + "onday": 29864, + "onde": 44532, + "ondo": 29529, + "ondon": 42043, + "ondon": 11851, + "one": 1980, + "one": 637, + "onec": 27746, + "oned": 28012, + "oned": 4698, + "onedirection": 16245, + "onee": 44433, + "oneill": 44808, + "onelove": 47417, + "onent": 12147, + "onents": 11709, + "oneof": 48478, + "onep": 20440, + "onepiece": 43153, + "oneplus": 25981, + "oner": 30055, + "oner": 6071, + "oners": 12324, + "ones": 20757, + "ones": 1575, + "oneself": 46874, + "onesie": 33237, + "oness": 25379, + "onet": 36058, + "oneteam": 41094, + "onetsy": 33392, + "onew": 43848, + "onews": 18696, + "onex": 49116, + "oney": 44498, + "oney": 9408, + "onf": 41790, + "onfox": 29874, + "ong": 2787, + "ong": 846, + "onga": 30259, + "ongchang": 35071, + "ongi": 21754, + "ongo": 31226, + "ongoing": 10393, + "ongs": 12143, + "oni": 4385, + "oni": 8048, + "onia": 8001, + "onial": 27599, + "onian": 21090, + "onic": 15838, + "onic": 3711, + "onica": 14631, + "onics": 9779, + "onie": 35249, + "onies": 22601, + "onimo": 41271, + "oning": 5197, + "onion": 10985, + "onions": 15255, + "onist": 10099, + "onists": 19659, + "onix": 27370, + "onized": 43657, + "onlin": 31103, + "online": 12940, + "online": 2027, + "onlinemarketing": 41820, + "onlineshopping": 38587, + "only": 11646, + "only": 1033, + "onlyin": 32947, + "onna": 25438, + "onna": 35458, + "onnaise": 48934, + "onne": 23466, + "onnell": 45613, + "ono": 28165, + "ono": 14388, + "onom": 48014, + "onomy": 36873, + "onpoli": 20708, + "ons": 26076, + "ons": 708, + "onsale": 36324, + "onset": 30527, + "onsite": 37336, + "onstage": 21821, + "onstorm": 49333, + "ont": 34303, + "ont": 11157, + "ontari": 6739, + "ontario": 42766, + "ontario": 7436, + "onte": 34723, + "onthe": 12241, + "onther": 46563, + "ontheroad": 47516, + "onthisday": 6862, + "onto": 11745, + "onto": 3141, + "ontology": 37364, + "ontour": 32155, + "onu": 44142, + "onward": 34827, + "onwards": 20682, + "ony": 9490, + "ony": 2926, + "onym": 11483, + "onymous": 13038, + "onyx": 31353, + "oo": 574, + "oo": 2822, + "ood": 16429, + "ood": 738, + "oodle": 45289, + "oods": 44660, + "oof": 42270, + "ooh": 16806, + "ook": 22326, + "ook": 8394, + "ooks": 31082, + "ool": 37702, + "ool": 929, + "oom": 22786, + "oom": 15002, + "oomf": 40607, + "oon": 35651, + "oon": 7100, + "ooo": 9571, + "oooh": 28927, + "oooo": 4002, + "oooo": 13643, + "ooooo": 12532, + "oooooo": 43590, + "oooooo": 20372, + "ooooooo": 30859, + "oooooooo": 15473, + "oooooooo": 43408, + "oooooooooooooooo": 48645, + "oop": 7326, + "ooper": 39906, + "oops": 9116, + "oor": 35239, + "oos": 9896, + "oosa": 30834, + "oose": 38941, + "oot": 17667, + "ootball": 28914, + "ootd": 16547, + "ooth": 12682, + "oott": 34316, + "ooza": 22809, + "op": 676, + "op": 3691, + "opa": 28949, + "opal": 28982, + "opar": 18167, + "opath": 33079, + "opathic": 37521, + "opathy": 28466, + "opau": 27239, + "opd": 38288, + "ope": 31694, + "ope": 11440, + "opec": 33138, + "opel": 36952, + "open": 3647, + "open": 1488, + "openaccess": 26591, + "opend": 28069, + "opendata": 35709, + "openday": 46991, + "opened": 5303, + "opener": 8998, + "openhouse": 36091, + "opening": 33728, + "opening": 2516, + "openingday": 36359, + "openings": 27643, + "openly": 23005, + "opens": 4801, + "opensource": 29930, + "oper": 2796, + "oper": 37533, + "opera": 8056, + "operate": 19306, + "operated": 23031, + "operates": 38675, + "operating": 12218, + "operation": 27173, + "operation": 7639, + "operational": 18237, + "operations": 8106, + "operative": 28380, + "operator": 15972, + "operators": 19267, + "opers": 48728, + "opes": 37258, + "oph": 6796, + "opha": 38634, + "ophel": 45017, + "ophelia": 49118, + "ophi": 44547, + "ophile": 35915, + "opho": 12900, + "ophobia": 21111, + "ophobic": 29934, + "ophon": 25120, + "ophone": 26345, + "ophthal": 33135, + "ophy": 28539, + "opi": 40056, + "opi": 48994, + "opin": 7636, + "opini": 14825, + "opinion": 7843, + "opinions": 16192, + "opio": 17371, + "opioid": 22833, + "opioids": 47578, + "opla": 36270, + "ople": 25663, + "opol": 15173, + "opoly": 23729, + "opor": 39650, + "opoulos": 42020, + "opp": 2020, + "opp": 21024, + "oppa": 23637, + "oppo": 7399, + "oppo": 41770, + "opponent": 17002, + "opponents": 19664, + "oppor": 2914, + "opportun": 2939, + "opportunities": 5978, + "opportunity": 4004, + "oppos": 10091, + "oppose": 23617, + "opposed": 22509, + "opposes": 47471, + "opposing": 24376, + "opposite": 12872, + "opposition": 11062, + "oppre": 17341, + "oppressed": 41492, + "oppression": 30650, + "opra": 28291, + "oprah": 22562, + "opry": 35340, + "ops": 3054, + "opt": 45103, + "opt": 27188, + "opted": 42035, + "opti": 6580, + "optic": 25190, + "optic": 24755, + "optical": 16822, + "optics": 27165, + "optim": 22331, + "optimal": 25235, + "optimi": 9737, + "optimis": 39459, + "optimism": 25226, + "optimist": 44581, + "optimistic": 23104, + "optimization": 25125, + "optimize": 30456, + "optimized": 43939, + "optimizing": 49157, + "optimum": 35974, + "optimus": 43453, + "option": 8464, + "optional": 25411, + "options": 7063, + "optome": 35533, + "opul": 39858, + "opus": 33295, + "opy": 21835, + "or": 523, + "or": 541, + "ora": 4301, + "orac": 24673, + "oracle": 37308, + "oracle": 15966, + "orah": 40820, + "orail": 45120, + "oral": 32490, + "oral": 6007, + "orama": 33619, + "oran": 32209, + "oran": 28395, + "orang": 22116, + "orange": 13957, + "orange": 4287, + "oranges": 32417, + "orangu": 36112, + "orb": 28894, + "orb": 36958, + "orbit": 19713, + "orbital": 40312, + "orc": 44305, + "orca": 18631, + "orcas": 47676, + "orch": 11893, + "orchar": 40226, + "orchard": 19530, + "orche": 8004, + "orchestr": 42937, + "orchestra": 9573, + "orchestral": 40285, + "orchi": 23696, + "orchid": 18678, + "orchids": 28376, + "ord": 26903, + "ord": 11502, + "orda": 33462, + "ordained": 38302, + "order": 24613, + "order": 2191, + "ordered": 8335, + "ordering": 19588, + "orderly": 43457, + "orders": 6187, + "ordin": 4378, + "ordinance": 38583, + "ordinary": 8012, + "ore": 3580, + "ore": 1423, + "orean": 36696, + "ored": 5133, + "oregon": 21759, + "oregon": 8035, + "oren": 21645, + "oreo": 21873, + "oreos": 41688, + "ores": 17328, + "org": 3401, + "org": 5593, + "organ": 3338, + "organ": 13213, + "organi": 3636, + "organic": 24080, + "organic": 5980, + "organics": 44199, + "organis": 13204, + "organisation": 15868, + "organisations": 20651, + "organise": 36073, + "organised": 13191, + "organiser": 49141, + "organisers": 35778, + "organising": 22787, + "organisms": 37041, + "organiz": 11107, + "organization": 8064, + "organizational": 29510, + "organizations": 13453, + "organize": 19973, + "organized": 10681, + "organizer": 23905, + "organizers": 27191, + "organizing": 15779, + "organs": 29872, + "orgs": 29500, + "ori": 1540, + "ori": 8693, + "oria": 11474, + "orial": 8648, + "orian": 21193, + "oric": 43810, + "orice": 41341, + "orie": 18815, + "orient": 13149, + "orient": 30770, + "oriental": 23056, + "orientation": 16873, + "oriente": 40390, + "oriented": 24596, + "orienteering": 42985, + "ories": 5934, + "orig": 2273, + "orig": 38463, + "origami": 31832, + "origin": 2555, + "origin": 12372, + "original": 18496, + "original": 3117, + "originally": 12849, + "originals": 16953, + "originated": 41823, + "origins": 16291, + "orin": 39863, + "oring": 3006, + "orio": 24308, + "orioles": 21430, + "orion": 21765, + "oris": 37064, + "orities": 7903, + "ority": 5556, + "orium": 12015, + "ork": 22202, + "ork": 37235, + "orkney": 34254, + "orl": 39465, + "orlando": 32247, + "orlando": 7827, + "orleans": 11127, + "orm": 38464, + "orn": 25412, + "orn": 8130, + "ornam": 36122, + "ornament": 23409, + "ornamental": 46270, + "ornaments": 28968, + "ornate": 46865, + "orni": 27713, + "ornithology": 38275, + "orns": 19340, + "oro": 9848, + "oro": 14573, + "orous": 19286, + "orph": 17318, + "orphan": 22718, + "orphan": 28994, + "orphanage": 45196, + "orphaned": 46792, + "orphans": 36588, + "orphe": 39186, + "orr": 32977, + "ors": 1127, + "orship": 20846, + "ort": 1019, + "ortega": 39727, + "orth": 22584, + "orth": 24461, + "ortho": 11366, + "orthodon": 37730, + "orthodox": 19008, + "orthop": 42123, + "orthopedic": 49341, + "ortiz": 23544, + "orton": 37238, + "oru": 44629, + "oru": 31281, + "orum": 42724, + "orwell": 41218, + "ory": 16983, + "ory": 1985, + "os": 2211, + "os": 1299, + "osa": 16340, + "osa": 17237, + "osaka": 21347, + "osborne": 22402, + "osbourne": 43376, + "osc": 5092, + "oscar": 21157, + "oscar": 8191, + "oscars": 11098, + "osce": 37303, + "oscill": 38272, + "ose": 46942, + "ose": 22541, + "osh": 30717, + "osh": 35011, + "osha": 33907, + "oshi": 34770, + "osi": 25247, + "osi": 17636, + "osis": 13903, + "osity": 12730, + "oslo": 20547, + "osm": 31626, + "osman": 46539, + "oso": 42793, + "oso": 21285, + "osp": 24387, + "ospre": 49001, + "osprey": 37893, + "oss": 29362, + "oss": 34640, + "ost": 23701, + "ost": 18749, + "oste": 20632, + "osteo": 43163, + "oster": 31781, + "ostr": 33673, + "ostrich": 47640, + "osu": 29480, + "osu": 19818, + "oswald": 38471, + "ot": 1863, + "ot": 2062, + "ota": 17509, + "ota": 8741, + "otago": 45919, + "otaku": 40743, + "otas": 47616, + "otc": 37934, + "otd": 5683, + "ote": 28511, + "ote": 19744, + "otes": 27280, + "oth": 33262, + "oth": 33519, + "other": 9758, + "other": 1010, + "others": 3326, + "otherwise": 12376, + "oti": 19567, + "oti": 45564, + "otic": 9671, + "otis": 28246, + "otive": 10877, + "oto": 23946, + "oto": 23399, + "otp": 29822, + "otr": 38685, + "ots": 5769, + "ott": 10167, + "ott": 7936, + "otta": 7623, + "otta": 20941, + "ottawa": 49027, + "ottawa": 9019, + "otte": 35214, + "otter": 34710, + "otter": 22456, + "otters": 38883, + "otti": 36721, + "ottnews": 33995, + "otto": 17730, + "ottoman": 27503, + "otw": 35259, + "otwol": 46868, + "ou": 520, + "ou": 6544, + "ouat": 32954, + "ouch": 13493, + "oud": 1359, + "oue": 48838, + "ouf": 34618, + "ough": 4204, + "ough": 991, + "ought": 2253, + "oughton": 36860, + "oui": 39421, + "ouk": 21796, + "oul": 20253, + "oul": 8081, + "ould": 859, + "oulos": 32808, + "oun": 636, + "oun": 20960, + "ounce": 15027, + "ounces": 30299, + "ound": 2013, + "ound": 853, + "oundation": 40132, + "ounded": 9634, + "ounding": 11944, + "ounds": 2753, + "oung": 35875, + "oung": 25341, + "ounge": 29427, + "ount": 43801, + "ount": 4172, + "ounts": 10963, + "oup": 32815, + "our": 727, + "our": 581, + "oura": 29806, + "oura": 36352, + "ourable": 24126, + "ourage": 34525, + "oural": 45840, + "oured": 6956, + "ouri": 12696, + "ouring": 12000, + "ourism": 25496, + "ourke": 26480, + "ourlives": 37541, + "ouro": 41224, + "ours": 1491, + "ourse": 15415, + "ourselves": 10124, + "ourt": 22960, + "oury": 29484, + "ous": 1987, + "ous": 879, + "ouse": 32048, + "ouse": 7603, + "ouses": 33666, + "ously": 2501, + "ousness": 10689, + "ousy": 28302, + "out": 1130, + "out": 620, + "outa": 35187, + "outage": 27320, + "outages": 40353, + "outback": 28532, + "outbound": 41256, + "outbreak": 20103, + "outcome": 16552, + "outcomes": 14016, + "outdated": 38313, + "outdoor": 19184, + "outdoor": 6368, + "outdoors": 10469, + "oute": 44180, + "outed": 34435, + "outer": 30499, + "outer": 14188, + "outes": 39600, + "outfield": 41826, + "outfit": 6525, + "outfits": 16366, + "outfitters": 37725, + "outfy": 34920, + "outgoing": 27302, + "outh": 16933, + "outh": 8111, + "outine": 35452, + "outing": 11251, + "outlander": 45820, + "outlander": 17095, + "outlaw": 37498, + "outlaw": 27340, + "outlaws": 30935, + "outlet": 16855, + "outlets": 20822, + "outline": 26894, + "outlines": 29159, + "outlining": 45960, + "outlook": 12983, + "outof": 43958, + "outpatient": 46603, + "outpost": 44622, + "output": 17255, + "outra": 14262, + "outrage": 23577, + "outraged": 43402, + "outrageous": 29342, + "outre": 14373, + "outreach": 15297, + "outright": 38200, + "outs": 5790, + "outsi": 22515, + "outside": 47693, + "outside": 2782, + "outsider": 41196, + "outsiders": 41742, + "outskirts": 42088, + "outsourcing": 34543, + "outstanding": 6387, + "outta": 15807, + "outtuesday": 48692, + "outw": 34650, + "oux": 40960, + "oux": 14228, + "ov": 6420, + "ov": 8479, + "ova": 12762, + "oval": 15039, + "ovarian": 42913, + "ovation": 24333, + "ove": 8649, + "ove": 15456, + "oven": 44620, + "oven": 12579, + "over": 1658, + "over": 962, + "overall": 6914, + "overboard": 42982, + "overcame": 47235, + "overcast": 36942, + "overcome": 14365, + "overcoming": 29348, + "overdose": 27017, + "overdrive": 40088, + "overdue": 30240, + "overflow": 32885, + "overflowing": 45370, + "overhaul": 31531, + "overhead": 20321, + "overland": 38808, + "overlay": 44827, + "overload": 24327, + "overlook": 35767, + "overlooked": 27632, + "overlooking": 17319, + "overly": 28820, + "overnight": 9913, + "overpass": 44310, + "overrated": 38214, + "overs": 45774, + "overs": 17329, + "overseas": 15100, + "oversight": 32494, + "oversized": 31557, + "overtime": 19347, + "overturned": 31048, + "overview": 14789, + "overwatch": 18124, + "overweight": 43465, + "overwhel": 12204, + "overwhelmed": 23459, + "overwhelming": 20306, + "overwhelmingly": 43549, + "ovi": 32508, + "ovic": 22417, + "ovich": 27623, + "ovie": 47677, + "ovo": 41920, + "ovo": 18065, + "ovski": 26167, + "ow": 2032, + "ow": 2250, + "owa": 32770, + "owe": 19073, + "owed": 37641, + "owen": 24838, + "owen": 12056, + "owens": 20664, + "owes": 35069, + "owing": 48582, + "owl": 34332, + "owl": 9899, + "owls": 18247, + "own": 3845, + "own": 1758, + "owned": 8536, + "owner": 5019, + "owners": 7712, + "ownership": 16583, + "owning": 24661, + "owns": 17533, + "owo": 46142, + "ows": 27423, + "owski": 22573, + "ox": 3282, + "ox": 12071, + "oxfam": 45466, + "oxford": 28588, + "oxford": 8824, + "oxfordshire": 37855, + "oxi": 33731, + "oxi": 48147, + "oxid": 17701, + "oxide": 28235, + "oxo": 37088, + "oxy": 12432, + "oxygen": 16214, + "oy": 6638, + "oy": 12437, + "oya": 38894, + "oye": 48677, + "oyster": 40545, + "oyster": 17253, + "oysters": 22672, + "oz": 10584, + "oz": 6044, + "ozar": 31848, + "ozil": 41365, + "ozone": 37052, + "ozzy": 39549, + "p": 79, + "p": 335, + "pa": 765, + "pa": 2217, + "paa": 32812, + "pab": 9354, + "pablo": 42172, + "pablo": 14473, + "pac": 2332, + "pac": 7608, + "pace": 40600, + "pace": 9450, + "paced": 32611, + "pacers": 23976, + "paces": 43001, + "paci": 5699, + "pacific": 19723, + "pacific": 6654, + "pacing": 45202, + "pack": 2711, + "pack": 3420, + "package": 7053, + "packaged": 29656, + "packages": 14305, + "packaging": 11658, + "packard": 46421, + "packed": 5883, + "packer": 28209, + "packers": 14294, + "packet": 25022, + "packets": 40448, + "packing": 9829, + "packs": 11086, + "paco": 41364, + "pacqui": 28456, + "pacquiao": 30485, + "pact": 27182, + "pad": 3798, + "pad": 7601, + "padded": 42253, + "paddington": 33162, + "paddle": 38276, + "paddle": 20811, + "paddling": 40645, + "paddock": 29590, + "paddy": 33103, + "paddy": 19855, + "padi": 47037, + "padilla": 22380, + "padma": 44595, + "padma": 46457, + "padre": 38343, + "padres": 22829, + "pads": 17353, + "paedi": 41488, + "paella": 46924, + "paf": 47185, + "pafc": 49259, + "pag": 4151, + "pag": 30525, + "pagan": 27854, + "page": 14996, + "page": 2504, + "pageant": 22139, + "pages": 8082, + "pagoda": 44309, + "pah": 41054, + "pah": 26884, + "pai": 20624, + "pai": 21198, + "paid": 5057, + "paige": 33659, + "paige": 16022, + "paign": 31796, + "pain": 2141, + "pain": 4495, + "paine": 38069, + "painful": 16361, + "pains": 25639, + "paint": 7948, + "paint": 5185, + "paintball": 39730, + "painted": 6433, + "painter": 10888, + "painters": 35703, + "painting": 49164, + "painting": 3086, + "paintings": 9956, + "paints": 21672, + "pair": 19848, + "pair": 4038, + "paired": 12433, + "pairing": 16313, + "pairings": 41152, + "pairs": 9950, + "pais": 16878, + "paisley": 22954, + "pajam": 24110, + "pajama": 40244, + "pajamas": 37231, + "pak": 13186, + "pak": 9094, + "paki": 3438, + "pakistan": 10713, + "pakistan": 3994, + "pakistani": 14050, + "pakistanis": 45707, + "pakv": 38196, + "pal": 1850, + "pal": 3611, + "pala": 17895, + "palace": 6381, + "palaces": 45625, + "palad": 28371, + "palae": 43379, + "palais": 35673, + "palate": 34666, + "palawan": 48202, + "palazzo": 36006, + "pale": 4768, + "pale": 12518, + "paleo": 36741, + "paleo": 22198, + "paler": 38028, + "palermo": 40635, + "palestin": 9449, + "palestine": 11682, + "palestinian": 11764, + "palestinians": 21874, + "palette": 13901, + "pali": 48063, + "palin": 40153, + "palis": 44256, + "pality": 27296, + "pall": 35817, + "palla": 21208, + "palladium": 37888, + "pallet": 39057, + "palli": 28954, + "palliative": 46014, + "pally": 46073, + "palm": 19651, + "palm": 8612, + "palma": 29888, + "palmer": 40112, + "palmer": 13633, + "palms": 27059, + "palo": 31562, + "palom": 47698, + "palooza": 25861, + "pals": 11043, + "palsy": 46651, + "pam": 8228, + "pam": 18513, + "pamela": 26991, + "pamp": 37653, + "pamper": 44345, + "pamph": 41332, + "pan": 1072, + "pan": 7437, + "panam": 24606, + "panama": 15522, + "panas": 26207, + "panasonic": 29750, + "pancake": 18723, + "pancakes": 15308, + "panch": 27251, + "pancra": 42472, + "pancre": 27708, + "pancreatic": 49337, + "pancy": 41625, + "pand": 5631, + "panda": 12952, + "pandas": 35119, + "pande": 38419, + "pandey": 34895, + "pandit": 41191, + "pandor": 30250, + "pandora": 17727, + "pandoramusic": 42344, + "pane": 27470, + "panel": 3724, + "paneli": 19410, + "panelist": 39719, + "panelists": 24619, + "panels": 12735, + "panera": 48471, + "pang": 16756, + "pang": 23672, + "panhandle": 40919, + "pani": 36092, + "panic": 46671, + "panic": 14124, + "panini": 30410, + "pann": 42302, + "panna": 49065, + "pano": 36165, + "panor": 12962, + "panorama": 19763, + "panoramic": 22563, + "pans": 35204, + "pant": 22550, + "panther": 22825, + "panther": 13262, + "panthers": 10494, + "panties": 32515, + "panto": 28776, + "pantry": 25608, + "pants": 5003, + "panty": 44217, + "pany": 45567, + "panzer": 41159, + "pao": 33790, + "paola": 44689, + "paolo": 48488, + "paolo": 21133, + "pap": 1884, + "pap": 30756, + "papa": 12211, + "papar": 32782, + "paparazzi": 37842, + "papaya": 44098, + "paper": 8680, + "paper": 2802, + "paperback": 17928, + "papers": 8204, + "paperwork": 35785, + "papi": 35177, + "papp": 26361, + "paprika": 44793, + "papua": 32629, + "par": 699, + "par": 9163, + "para": 18355, + "para": 8976, + "parach": 23147, + "parachute": 30122, + "parad": 37143, + "parade": 5809, + "parades": 46479, + "paradi": 6658, + "paradig": 27786, + "paradigm": 33485, + "paradise": 45869, + "paradise": 7247, + "paradox": 33109, + "parag": 11866, + "paragon": 48099, + "paragra": 24903, + "paragraph": 28499, + "paragu": 38021, + "paraguay": 43579, + "paral": 15143, + "paralle": 13184, + "parallel": 18201, + "paralleled": 42520, + "parallels": 46101, + "paraly": 30255, + "paralym": 18727, + "paralympic": 30806, + "paralympics": 37162, + "paralysis": 45702, + "param": 12250, + "parame": 27106, + "paramedic": 34630, + "paramedics": 35991, + "parameters": 44890, + "paramore": 34401, + "paramount": 26642, + "parano": 30283, + "paranoid": 43029, + "paranor": 16940, + "paranormal": 19047, + "parap": 41091, + "paras": 15198, + "parasite": 42460, + "parasites": 46175, + "parc": 30914, + "parcel": 30367, + "parcels": 45589, + "pard": 18773, + "pardon": 47606, + "pardon": 26565, + "pare": 18202, + "pared": 5498, + "paren": 3106, + "parent": 47848, + "parent": 10183, + "parental": 28339, + "parenthood": 23887, + "parenting": 14529, + "parents": 3731, + "pares": 12420, + "parfait": 46140, + "pari": 17961, + "pari": 27979, + "paris": 13982, + "paris": 3445, + "parisagreement": 47405, + "parish": 47328, + "parish": 13020, + "parisi": 45081, + "parisian": 38512, + "parity": 42734, + "park": 4985, + "park": 1452, + "parked": 16487, + "parker": 31119, + "parker": 8365, + "parkin": 34868, + "parking": 5984, + "parkinson": 28129, + "parkland": 31287, + "parkrun": 25747, + "parks": 6873, + "parkway": 19882, + "parl": 30373, + "parl": 29897, + "parliam": 5941, + "parliament": 41599, + "parliament": 7151, + "parliamentary": 17912, + "parlor": 38253, + "parlour": 37829, + "parma": 36077, + "parme": 26295, + "parmesan": 27274, + "paro": 17429, + "parody": 24318, + "parole": 32158, + "parr": 44113, + "parrish": 43043, + "parrot": 23565, + "parry": 40604, + "parsley": 30077, + "parsons": 22505, + "part": 1872, + "part": 1551, + "parte": 48508, + "parth": 34790, + "parti": 10509, + "partial": 18957, + "partially": 21269, + "partic": 2871, + "partici": 9540, + "particip": 4400, + "participant": 27674, + "participants": 10237, + "participate": 9433, + "participated": 14252, + "participates": 46414, + "participating": 11535, + "participation": 13529, + "particle": 27716, + "particles": 27012, + "particul": 11408, + "particular": 14098, + "particularly": 12170, + "parties": 9032, + "parting": 32844, + "partisan": 20772, + "partist": 44713, + "partition": 42219, + "partly": 21459, + "partner": 5210, + "partner": 4568, + "partnered": 21402, + "partnering": 21182, + "partners": 5568, + "partnership": 6123, + "partnerships": 17418, + "parton": 43245, + "partridge": 34872, + "parts": 5149, + "party": 12877, + "party": 1580, + "partying": 25702, + "pas": 1341, + "pas": 9525, + "pasadena": 25892, + "pascal": 28626, + "pasco": 49220, + "pascu": 42692, + "pash": 23936, + "pasha": 46986, + "paso": 18542, + "pasqu": 44941, + "pass": 5016, + "pass": 3511, + "passage": 16477, + "passages": 48937, + "passed": 4957, + "passenger": 12311, + "passengers": 12781, + "passer": 48544, + "passes": 7633, + "passi": 32471, + "passing": 6589, + "passion": 8822, + "passion": 5332, + "passionate": 10947, + "passionately": 44028, + "passions": 38441, + "passive": 23171, + "passover": 38426, + "passport": 14739, + "passports": 46368, + "password": 20258, + "passwords": 43095, + "past": 7315, + "past": 2729, + "pasta": 10441, + "paste": 34765, + "paste": 17038, + "pastel": 19457, + "pastels": 45699, + "pastor": 19792, + "pastor": 9664, + "pastoral": 37191, + "pastors": 30959, + "pastr": 45478, + "pastries": 39409, + "pastry": 18582, + "pasture": 34764, + "pastures": 47793, + "pat": 1300, + "pat": 7036, + "patag": 29862, + "patagonia": 32786, + "patch": 29284, + "patch": 8721, + "patches": 22104, + "patchwork": 44675, + "patchy": 47488, + "pate": 42122, + "pate": 42098, + "patel": 14168, + "patent": 14692, + "patented": 37277, + "patents": 33911, + "paterson": 36560, + "path": 7408, + "path": 5035, + "pathetic": 18222, + "pathfinder": 35415, + "pathi": 34976, + "pathi": 27347, + "pathic": 49025, + "patho": 18534, + "pathology": 23290, + "paths": 16333, + "pathway": 23488, + "pathways": 24690, + "pathy": 13330, + "pati": 2799, + "pati": 26708, + "patience": 13575, + "patient": 30139, + "patient": 6262, + "patiently": 22980, + "patients": 5543, + "patil": 49187, + "patio": 14304, + "pational": 30627, + "patna": 45025, + "patory": 41859, + "patreon": 17165, + "patri": 4771, + "patriarch": 49054, + "patriarchy": 48806, + "patric": 12569, + "patrice": 40731, + "patricia": 18143, + "patrick": 12078, + "patrick": 5286, + "patricks": 46783, + "patriot": 28896, + "patriot": 15692, + "patrioti": 35520, + "patriotic": 20217, + "patriotism": 35807, + "patriots": 8707, + "patro": 31650, + "patrol": 10073, + "patrolling": 39344, + "patrols": 35978, + "patron": 26658, + "patron": 17683, + "patrons": 28308, + "pats": 24874, + "patsy": 46093, + "patt": 12637, + "patter": 4982, + "pattern": 7447, + "patterned": 47212, + "patterns": 11637, + "patterson": 21384, + "patti": 44927, + "patti": 26123, + "pattinson": 32474, + "patton": 29026, + "patty": 48741, + "patty": 18321, + "pau": 1834, + "pau": 35970, + "paul": 6035, + "paul": 2597, + "paula": 37363, + "paula": 16777, + "pauline": 30438, + "paulo": 48002, + "paulo": 21628, + "pauls": 41413, + "pauls": 40010, + "paulson": 48201, + "pause": 19439, + "paused": 46782, + "pav": 6661, + "pave": 37107, + "paved": 27898, + "pavel": 43152, + "pavement": 27669, + "pavilion": 13374, + "paving": 28651, + "paw": 14009, + "paw": 16016, + "pawan": 29754, + "pawankalyan": 33702, + "pawn": 43195, + "paws": 16714, + "pax": 20007, + "pax": 19033, + "paxton": 38347, + "pay": 2642, + "pay": 3345, + "payback": 36413, + "paycheck": 45078, + "payday": 26957, + "payee": 46985, + "payer": 41503, + "paying": 8341, + "payment": 10596, + "payments": 11832, + "payne": 12775, + "paypal": 21442, + "payroll": 31610, + "pays": 10845, + "paysoff": 48174, + "paytm": 45352, + "payton": 27348, + "paz": 22267, + "pb": 20112, + "pb": 10981, + "pba": 28205, + "pbb": 48567, + "pbb": 40589, + "pbc": 49191, + "pbl": 35166, + "pbr": 32998, + "pbs": 17908, + "pc": 6782, + "pc": 3808, + "pca": 35705, + "pcb": 26235, + "pcc": 36059, + "pci": 38957, + "pcm": 47436, + "pcr": 35704, + "pcs": 11917, + "pcso": 31963, + "pct": 22168, + "pd": 4387, + "pd": 4675, + "pdates": 16842, + "pdc": 40498, + "pdf": 15181, + "pdp": 24601, + "pdt": 21743, + "pdx": 25470, + "pdx": 16153, + "pe": 661, + "pe": 956, + "pea": 13915, + "peabo": 34083, + "peabody": 41244, + "peac": 34615, + "peace": 6249, + "peace": 3021, + "peaceful": 9461, + "peacefully": 30530, + "peacekeeping": 43630, + "peach": 10522, + "peach": 11538, + "peaches": 27216, + "peak": 18572, + "peak": 6026, + "peakdistrict": 41289, + "peake": 24810, + "peaked": 36391, + "peaks": 14067, + "pean": 11563, + "peanu": 25843, + "peanut": 12491, + "peanuts": 26503, + "pear": 4910, + "pear": 18820, + "pearce": 25996, + "pearl": 21806, + "pearl": 8560, + "pearljam": 46739, + "pearls": 19581, + "pears": 39565, + "pearson": 20461, + "peas": 15937, + "peasant": 40621, + "peasants": 48788, + "peat": 26914, + "pebble": 28056, + "pebbles": 40155, + "pec": 32447, + "pec": 17611, + "pecan": 32177, + "peck": 25186, + "peck": 29234, + "pecker": 30169, + "peckham": 45863, + "pecu": 34200, + "peculiar": 42808, + "ped": 13197, + "ped": 2966, + "pedago": 34590, + "pedagogy": 48072, + "pedal": 32943, + "pedal": 19621, + "pedals": 38535, + "pede": 12862, + "pede": 19560, + "pedestri": 30027, + "pedestrian": 18256, + "pedestrians": 33895, + "pedi": 12967, + "pedia": 11733, + "pediatric": 48431, + "pediatric": 22071, + "pedic": 35319, + "pedic": 44528, + "pedro": 29963, + "pedro": 15114, + "peds": 45377, + "pee": 12988, + "pee": 11196, + "peed": 47369, + "peek": 46323, + "peek": 7569, + "peeking": 48771, + "peel": 34386, + "peel": 17158, + "peeled": 33533, + "peeling": 48649, + "peep": 25425, + "peep": 16857, + "peeps": 11681, + "peer": 32416, + "peer": 14432, + "peers": 21626, + "pees": 31830, + "peg": 32182, + "peg": 11207, + "pegas": 30018, + "pegasus": 37822, + "peggy": 24271, + "pei": 48166, + "pei": 12917, + "pel": 4286, + "pel": 7006, + "pele": 44105, + "pelican": 34131, + "pelicans": 29363, + "pell": 46981, + "pelle": 31267, + "pelled": 32506, + "pellegr": 38529, + "pellets": 48240, + "pelo": 40192, + "pelo": 40238, + "pelosi": 22169, + "pelvic": 45646, + "pemb": 19880, + "pembro": 24084, + "pembroke": 36702, + "pembroke": 40044, + "pembrokeshire": 40695, + "pen": 1501, + "pen": 5356, + "pena": 35788, + "penalties": 25417, + "penalty": 11491, + "penang": 29545, + "penc": 20065, + "pence": 18002, + "pencil": 41303, + "pencil": 11200, + "pencils": 21909, + "pend": 3052, + "pendant": 12415, + "pendants": 44117, + "pending": 12770, + "pendleton": 44272, + "pendu": 45336, + "penelope": 36703, + "penetr": 26058, + "peng": 42955, + "peng": 39200, + "pengu": 8854, + "penguin": 28249, + "penguin": 14952, + "penguins": 16557, + "peninsu": 13464, + "peninsula": 14070, + "penn": 7760, + "penn": 11128, + "pennant": 43971, + "penned": 45077, + "penney": 47856, + "pennies": 43094, + "pennsylvania": 13673, + "penny": 20400, + "penny": 11388, + "pens": 13307, + "pens": 13310, + "pensac": 30925, + "pensacola": 33573, + "pension": 32840, + "pension": 17764, + "pensions": 29773, + "penske": 47154, + "pent": 10699, + "pent": 22725, + "pentagon": 23133, + "pente": 33165, + "penthouse": 32673, + "penultimate": 36553, + "peop": 1030, + "people": 10573, + "people": 1047, + "peoples": 28241, + "peoples": 14627, + "peopleschoice": 32418, + "peoplesvote": 45830, + "peoria": 36985, + "pep": 12761, + "pep": 14898, + "pepe": 24778, + "pepp": 34425, + "pepper": 14861, + "pepper": 8253, + "peppermint": 30321, + "pepperoni": 47307, + "peppers": 14650, + "pepsi": 21307, + "per": 703, + "per": 1284, + "pera": 26294, + "perce": 24135, + "perceived": 38436, + "percent": 16328, + "percent": 9017, + "percentage": 19477, + "percep": 28017, + "perception": 20591, + "perceptions": 38138, + "perch": 34281, + "perched": 40071, + "percu": 41722, + "percussion": 23980, + "percy": 23940, + "pere": 8665, + "pere": 36300, + "pered": 24509, + "peregr": 37479, + "peregrine": 44546, + "pereira": 43927, + "peren": 24564, + "perenni": 26996, + "perennial": 34038, + "perez": 15107, + "perf": 22816, + "perfe": 1624, + "perfec": 6599, + "perfect": 17261, + "perfect": 1878, + "perfection": 9646, + "perfectly": 8037, + "perfecto": 42898, + "perfor": 2311, + "perform": 3866, + "perform": 5940, + "performan": 8973, + "performance": 2714, + "performances": 9553, + "performed": 9997, + "performer": 17061, + "performers": 18476, + "performing": 5170, + "performs": 13839, + "perfu": 14214, + "perfume": 17525, + "perhaps": 9297, + "peri": 12618, + "peri": 44068, + "perience": 19302, + "peril": 40119, + "peril": 48301, + "perimeter": 38499, + "pering": 29746, + "perio": 5101, + "period": 6131, + "periodic": 36476, + "periods": 24401, + "periph": 35308, + "peripheral": 43901, + "peris": 19461, + "periscope": 21668, + "perk": 33424, + "perkins": 20057, + "perks": 17660, + "perl": 44018, + "perm": 47847, + "perman": 9018, + "permanent": 11144, + "permanently": 25584, + "perme": 42456, + "permission": 15822, + "permit": 21950, + "permits": 33267, + "permitted": 44380, + "pero": 23551, + "perpe": 15749, + "perpetr": 33376, + "perpetu": 30132, + "perpetual": 32018, + "perrie": 32691, + "perry": 28478, + "perry": 7899, + "pers": 3688, + "pers": 10710, + "perse": 27498, + "persecu": 22878, + "persecution": 32009, + "perseverance": 29820, + "persi": 11509, + "persian": 19859, + "persist": 19412, + "persist": 40938, + "persistence": 34588, + "persistent": 29028, + "person": 3510, + "person": 2533, + "persona": 18401, + "personal": 10114, + "personal": 4121, + "personalised": 24186, + "personalities": 27888, + "personality": 10386, + "personalized": 17845, + "personally": 13885, + "personnel": 14546, + "persons": 14592, + "perspec": 17997, + "perspective": 8996, + "perspectives": 18777, + "persu": 20972, + "pert": 36970, + "pert": 16306, + "perth": 19067, + "perth": 11011, + "peru": 20612, + "peru": 12964, + "peruvian": 30822, + "pes": 38368, + "pes": 2598, + "pesa": 47409, + "pesc": 44044, + "pesh": 33184, + "peshaw": 28524, + "peshawar": 29230, + "pesky": 42512, + "pesos": 47872, + "pessi": 43902, + "pest": 20130, + "pest": 9425, + "pesticide": 48481, + "pesticides": 37868, + "pesto": 26186, + "pests": 41919, + "pet": 2167, + "pet": 3703, + "peta": 28785, + "petal": 38430, + "petal": 40469, + "petals": 26064, + "petday": 45314, + "pete": 14479, + "pete": 8571, + "peter": 5093, + "peter": 3696, + "peterborough": 26012, + "peters": 16336, + "petersburg": 21052, + "petersen": 39794, + "peterson": 16877, + "peth": 48920, + "petit": 36437, + "petit": 21276, + "petite": 27213, + "petition": 10975, + "petitions": 43536, + "petr": 29808, + "petra": 31300, + "petre": 47179, + "petri": 31831, + "petro": 8716, + "petrol": 18149, + "petroleum": 22063, + "petron": 42875, + "pets": 7663, + "pett": 27051, + "petti": 48001, + "petting": 44334, + "petty": 17324, + "peu": 21411, + "peuge": 22893, + "peugeot": 24129, + "pew": 21608, + "pew": 30783, + "pewdie": 41882, + "pewdiepie": 42563, + "pex": 43765, + "pey": 14966, + "pey": 30933, + "peyton": 49254, + "peyton": 20307, + "pez": 45798, + "pez": 10482, + "pf": 16680, + "pf": 12572, + "pfa": 47839, + "pfc": 35007, + "pff": 44121, + "pfi": 29810, + "pfw": 31229, + "pg": 12476, + "pg": 5211, + "pga": 13351, + "pgat": 36514, + "pgatour": 40094, + "pgh": 44862, + "pgh": 30031, + "pgs": 49204, + "ph": 745, + "ph": 2042, + "pha": 4443, + "pha": 26255, + "phal": 19962, + "phan": 8731, + "phan": 40126, + "phant": 36998, + "phantom": 37688, + "phantom": 14490, + "phar": 5570, + "phara": 35792, + "pharaoh": 40437, + "pharm": 45761, + "pharma": 17831, + "pharmac": 8193, + "pharmaceu": 19490, + "pharmaceutical": 25217, + "pharmaceuticals": 44623, + "pharmacist": 41024, + "pharmacists": 44337, + "pharmacy": 15293, + "pharo": 42308, + "pharoah": 49287, + "pharrell": 31316, + "phase": 8304, + "phases": 35337, + "phat": 42492, + "phc": 41102, + "phd": 20875, + "phd": 8472, + "phdchat": 39564, + "phdlife": 39638, + "phe": 4787, + "phe": 19853, + "pheasant": 41983, + "phee": 41292, + "phel": 23711, + "phelps": 27128, + "phen": 7718, + "pheno": 47336, + "phenom": 31673, + "phenom": 39618, + "phenomen": 11304, + "phenomena": 41538, + "phenomenal": 15035, + "phenomenon": 24464, + "pher": 9194, + "pher": 19828, + "phers": 29531, + "pherson": 36421, + "phew": 10295, + "phi": 2239, + "phi": 12220, + "phia": 9228, + "phic": 3977, + "phie": 30237, + "phies": 17062, + "phil": 2821, + "phil": 6199, + "phila": 47443, + "philadel": 9428, + "philadelphia": 9749, + "philanthro": 16587, + "philanthropist": 44153, + "philanthropy": 25047, + "philately": 33695, + "phile": 36543, + "philharmon": 25228, + "philharmonic": 31699, + "phili": 4277, + "philia": 46654, + "philip": 20748, + "philip": 11074, + "philipp": 5623, + "philipp": 47591, + "philippe": 20942, + "philippine": 17629, + "philippines": 8149, + "philips": 25175, + "phill": 42346, + "phill": 48272, + "philli": 6456, + "phillies": 18748, + "phillip": 48832, + "phillip": 19323, + "phillips": 11041, + "philly": 19545, + "philly": 7785, + "philos": 8395, + "philosop": 20349, + "philosoph": 10187, + "philosopher": 25220, + "philosophical": 32628, + "philosophy": 12213, + "phils": 38573, + "phin": 33816, + "phine": 40985, + "phins": 40210, + "phish": 36897, + "phishing": 36546, + "phl": 25603, + "pho": 816, + "pho": 22707, + "phobia": 28749, + "phoe": 22673, + "phoebe": 27582, + "phoeni": 6778, + "phoenix": 20615, + "phoenix": 7793, + "phol": 48140, + "phon": 19602, + "phon": 31115, + "phone": 15486, + "phone": 1951, + "phones": 6351, + "phony": 31925, + "phora": 31363, + "phosp": 22638, + "photo": 1153, + "photo": 1125, + "photobomb": 37075, + "photobook": 41894, + "photog": 28115, + "photogenic": 36108, + "photogra": 36754, + "photograph": 1688, + "photograph": 8853, + "photographed": 11573, + "photographer": 5748, + "photographers": 17141, + "photographic": 22053, + "photographing": 30074, + "photographs": 15759, + "photography": 33183, + "photography": 2108, + "photom": 32223, + "photoo": 11106, + "photooftheday": 11933, + "photos": 2479, + "photoshoot": 11121, + "photoshop": 12419, + "photoshopped": 35738, + "phouse": 27848, + "php": 17370, + "phra": 12777, + "phrase": 18809, + "phrases": 35264, + "phs": 16495, + "phu": 21274, + "phuket": 34028, + "phx": 35466, + "phx": 29507, + "phy": 6484, + "phy": 4292, + "phyl": 35600, + "phyllis": 37844, + "phys": 3734, + "phys": 37894, + "physi": 13782, + "physic": 46641, + "physical": 44127, + "physical": 6671, + "physically": 18105, + "physician": 21055, + "physicians": 26702, + "physicist": 29052, + "physics": 9369, + "physio": 29574, + "physio": 29177, + "physiology": 32349, + "physique": 42884, + "phyto": 42197, + "pi": 741, + "pi": 5357, + "pia": 8918, + "pian": 24637, + "pianist": 21048, + "piano": 49278, + "piano": 7894, + "pianos": 47904, + "piazza": 28496, + "pic": 901, + "pic": 1282, + "pical": 5482, + "picard": 48507, + "picasso": 21481, + "piccad": 33876, + "piccadilly": 37287, + "piccollage": 43621, + "pick": 6379, + "pick": 3142, + "picked": 6018, + "picker": 43105, + "pickering": 47605, + "picket": 33559, + "picking": 9545, + "pickle": 24570, + "pickled": 21705, + "pickles": 25001, + "picks": 8551, + "pickup": 15382, + "pickups": 33383, + "picnic": 12007, + "pico": 23363, + "picoftheday": 18319, + "pics": 2559, + "pict": 18778, + "pictorial": 40640, + "picture": 11663, + "picture": 1674, + "pictured": 7647, + "pictures": 3646, + "picturesque": 24894, + "pid": 5225, + "piday": 48056, + "pie": 12065, + "pie": 5319, + "piece": 39632, + "piece": 2754, + "pieces": 6194, + "pied": 24686, + "pied": 12713, + "piedmont": 39691, + "pier": 5641, + "pier": 11348, + "pierc": 49216, + "pierce": 48462, + "pierce": 16782, + "pierced": 32799, + "piercing": 22557, + "piero": 43125, + "pierre": 34670, + "pierre": 11985, + "piers": 29030, + "pies": 6898, + "pieter": 44801, + "pietro": 42169, + "piff": 40719, + "pig": 12009, + "pig": 9619, + "pigeon": 18008, + "pigeons": 32910, + "piggy": 28245, + "pigment": 40284, + "pigs": 16228, + "pik": 48539, + "pika": 47372, + "pikach": 27268, + "pikachu": 28107, + "pike": 33457, + "pike": 14011, + "pil": 2893, + "pil": 20645, + "pilates": 29518, + "pile": 44403, + "pile": 13930, + "piled": 26873, + "piles": 31968, + "pilgri": 13966, + "pilgrim": 32662, + "pilgrimage": 24335, + "pilgrims": 31370, + "piling": 43050, + "pilip": 27234, + "pilipinas": 32392, + "pill": 14830, + "pill": 19226, + "pillar": 17322, + "pillars": 22054, + "pillow": 42237, + "pillow": 12182, + "pillows": 26499, + "pills": 23964, + "pilo": 37526, + "pilot": 31619, + "pilot": 6687, + "pilots": 15586, + "pilsner": 47153, + "pim": 15285, + "pim": 35472, + "pimp": 35789, + "pin": 2629, + "pin": 5164, + "pinball": 31679, + "pinch": 26114, + "pine": 9398, + "pine": 7374, + "pineapple": 14831, + "pines": 20338, + "ping": 23720, + "ping": 2089, + "pinion": 40557, + "pink": 11151, + "pink": 3360, + "pinkfloyd": 48520, + "pinky": 29803, + "pinn": 31448, + "pinnacle": 32754, + "pinned": 12165, + "pinning": 44515, + "pino": 36633, + "pinot": 41399, + "pinot": 21146, + "pinoy": 43578, + "pinoy": 35258, + "pins": 14619, + "pinst": 41173, + "pint": 42537, + "pint": 13584, + "pinterest": 15379, + "pinto": 35992, + "pints": 27935, + "pinup": 37349, + "pio": 22108, + "pion": 36728, + "pion": 29190, + "pione": 7975, + "pioneer": 34892, + "pioneer": 12459, + "pioneering": 25933, + "pioneers": 22383, + "pious": 42441, + "pip": 30854, + "pipe": 29333, + "pipe": 10459, + "pipel": 12387, + "pipeline": 14151, + "pipelines": 39683, + "piper": 47052, + "piper": 16293, + "pipes": 16991, + "piping": 40744, + "pippa": 47672, + "pir": 4351, + "pir": 38899, + "piracy": 39452, + "piran": 49034, + "pirate": 38680, + "pirate": 13592, + "pirates": 10442, + "pire": 16613, + "pires": 14988, + "pis": 9230, + "pis": 44441, + "pisa": 43632, + "pisces": 45982, + "piss": 20818, + "pissed": 17989, + "pist": 15556, + "pist": 32826, + "pistachi": 29760, + "pistachio": 36320, + "pistol": 20480, + "piston": 48236, + "pistons": 27242, + "pistor": 48162, + "pit": 2946, + "pit": 7476, + "pita": 27070, + "pitbull": 25295, + "pitch": 8992, + "pitch": 5872, + "pitched": 28447, + "pitcher": 13445, + "pitchers": 27835, + "pitches": 21005, + "pitching": 16455, + "piti": 47568, + "pits": 24144, + "pitt": 7607, + "pitt": 15599, + "pitts": 9531, + "pittsburgh": 10453, + "pity": 24380, + "pius": 39988, + "pivo": 18009, + "pivot": 31805, + "pivotal": 31432, + "pix": 6185, + "pix": 13088, + "pixar": 27493, + "pixel": 14384, + "pixel": 13241, + "pixelart": 18516, + "pixels": 34099, + "pixie": 35573, + "piyu": 30772, + "piyush": 36191, + "piyushgoyal": 45318, + "pizz": 3897, + "pizza": 4474, + "pizzas": 30647, + "pizzeria": 44174, + "pj": 12524, + "pj": 17179, + "pjnet": 22011, + "pjs": 36009, + "pk": 10149, + "pk": 10991, + "pkg": 49011, + "pkk": 47480, + "pknot": 41779, + "pkwy": 36827, + "pl": 712, + "pl": 5678, + "pla": 841, + "pla": 19945, + "plac": 2331, + "place": 14884, + "place": 1445, + "placed": 9729, + "placement": 16724, + "placements": 43885, + "placer": 49170, + "places": 4448, + "placing": 18531, + "plague": 25360, + "plaid": 23291, + "plain": 22776, + "plain": 10709, + "plains": 16345, + "plan": 1740, + "plan": 2970, + "pland": 24801, + "plane": 22728, + "plane": 5363, + "planes": 12581, + "planet": 16833, + "planet": 5172, + "planetary": 28361, + "planets": 22315, + "plank": 30991, + "plankton": 48249, + "plann": 6409, + "planned": 8169, + "planner": 18083, + "planners": 33664, + "planning": 4446, + "plano": 34063, + "plans": 4181, + "plant": 8521, + "plant": 3912, + "plantation": 20014, + "plantbased": 33720, + "planted": 14286, + "planter": 34453, + "planters": 43661, + "planting": 13922, + "plants": 5829, + "plaque": 16097, + "plaques": 45610, + "plar": 26754, + "plas": 45673, + "plasma": 24999, + "plaster": 31980, + "plastic": 15645, + "plastic": 6102, + "plasticpollution": 47129, + "plastics": 20999, + "plasticsurgery": 48555, + "plat": 3172, + "plata": 46456, + "plate": 28744, + "plate": 5135, + "plateau": 29301, + "plated": 21161, + "plates": 11485, + "platform": 5549, + "platforms": 13551, + "platin": 10267, + "plating": 44564, + "platinum": 10979, + "plato": 41101, + "platoon": 41254, + "platt": 44459, + "platt": 40097, + "platte": 46785, + "platter": 29071, + "platz": 40878, + "plau": 39139, + "play": 1222, + "play": 1453, + "playa": 23756, + "playable": 33885, + "playback": 39194, + "playbook": 34856, + "playboy": 24383, + "played": 3432, + "player": 24503, + "player": 2477, + "players": 3030, + "playful": 23871, + "playground": 15861, + "playhouse": 23254, + "playin": 24674, + "playing": 47368, + "playing": 1629, + "playlist": 9180, + "playlists": 47183, + "playo": 5804, + "playoff": 9655, + "playoffs": 9548, + "plays": 5134, + "playstation": 11332, + "playtime": 43037, + "playwright": 32070, + "plaza": 8943, + "plc": 16827, + "ple": 926, + "ple": 1619, + "plea": 21956, + "plead": 47539, + "pleads": 31425, + "plear": 21362, + "pleas": 8481, + "pleas": 48740, + "pleasant": 12271, + "please": 41074, + "please": 1474, + "pleased": 6107, + "pleasing": 32893, + "pleasure": 5854, + "pleasures": 29513, + "pledge": 11507, + "pledged": 36799, + "pledges": 26746, + "pledis": 41202, + "plein": 43429, + "plenary": 19891, + "plenty": 7524, + "pler": 17677, + "ples": 6248, + "pless": 39821, + "pless": 17059, + "plets": 43230, + "plex": 23765, + "plex": 15241, + "pley": 19543, + "pli": 30001, + "pli": 45797, + "plic": 5806, + "plicity": 19823, + "plight": 40317, + "plin": 44531, + "plin": 32335, + "pline": 25376, + "pling": 12899, + "plings": 31184, + "pll": 47629, + "pll": 25266, + "pln": 48755, + "plo": 1778, + "plo": 43523, + "plor": 34695, + "plot": 9918, + "plots": 25672, + "plotting": 30751, + "plough": 33811, + "plow": 38363, + "pls": 5572, + "plu": 2052, + "plug": 12628, + "plugged": 23261, + "plugin": 31278, + "plugins": 48797, + "plugs": 28083, + "plum": 26267, + "plum": 16202, + "plumb": 21769, + "plumber": 43478, + "plumbing": 24647, + "plume": 39495, + "plun": 15122, + "plunge": 26506, + "plur": 44664, + "plus": 3097, + "plush": 18926, + "pluto": 26380, + "ply": 17249, + "ply": 28705, + "plying": 36071, + "plym": 11907, + "plymouth": 13786, + "plz": 10538, + "pm": 13699, + "pm": 990, + "pmi": 41206, + "pmln": 23208, + "pmo": 18782, + "pmoindia": 20374, + "pms": 44223, + "pn": 14431, + "pn": 13774, + "pnc": 37148, + "pne": 30966, + "pneu": 28714, + "pneumonia": 42906, + "png": 20992, + "pnp": 25972, + "pnpp": 42175, + "pnw": 31521, + "po": 628, + "po": 3057, + "poa": 43912, + "poached": 27665, + "poaching": 35140, + "poc": 13232, + "poc": 27780, + "pocaly": 37987, + "pocalypse": 42307, + "poche": 38336, + "poche": 39022, + "pocket": 29147, + "pocket": 8504, + "pockets": 19566, + "pocon": 41850, + "pod": 3583, + "pod": 7446, + "podcast": 39654, + "podcast": 4294, + "podcasting": 40106, + "podcasts": 19392, + "pode": 33368, + "poder": 24960, + "podernfamily": 26620, + "podi": 32853, + "podium": 14093, + "pods": 18776, + "poe": 4746, + "poe": 19254, + "poem": 9436, + "poems": 15577, + "poet": 41019, + "poet": 9872, + "poetic": 26365, + "poetry": 20192, + "poetry": 6038, + "poetryday": 39255, + "poets": 19804, + "pof": 40850, + "poff": 28236, + "pogba": 25998, + "poign": 29682, + "poignant": 32138, + "poin": 9074, + "point": 13280, + "point": 2301, + "pointe": 24631, + "pointed": 20703, + "pointer": 29883, + "pointers": 36760, + "pointing": 19233, + "pointless": 33586, + "points": 3396, + "pois": 17008, + "poise": 45087, + "poised": 27354, + "poison": 30722, + "poison": 17074, + "poisoned": 43624, + "poisoning": 25750, + "poisonous": 37131, + "pok": 15387, + "poke": 6892, + "poke": 23186, + "pokemon": 16239, + "pokemon": 9528, + "pokemongo": 23985, + "poker": 30735, + "poker": 11865, + "pokes": 40221, + "poking": 49169, + "poké": 20656, + "pokémon": 22066, + "pol": 977, + "pol": 7649, + "pola": 43876, + "poland": 9834, + "polar": 21432, + "polar": 12214, + "polari": 27919, + "polaris": 37965, + "polarized": 48437, + "polaro": 25237, + "polaroid": 30427, + "poldark": 41322, + "pole": 26682, + "pole": 8170, + "poles": 22585, + "poli": 9675, + "poli": 5414, + "polic": 16126, + "police": 15535, + "police": 2120, + "policeman": 37713, + "policemen": 47946, + "polici": 10819, + "policies": 10993, + "policing": 20969, + "policy": 30173, + "policy": 4660, + "polio": 30533, + "polis": 16133, + "polish": 46941, + "polish": 9632, + "polished": 21478, + "polishing": 43629, + "polit": 2247, + "politan": 15337, + "polite": 31497, + "politi": 40597, + "politic": 33333, + "political": 37744, + "political": 4197, + "politically": 24323, + "politician": 15960, + "politicians": 12914, + "politico": 39403, + "politics": 4929, + "polk": 33317, + "polka": 29476, + "poll": 7032, + "pollen": 27651, + "pollin": 19152, + "pollinators": 36599, + "polling": 18024, + "pollo": 42755, + "pollock": 37614, + "polls": 11813, + "pollu": 8370, + "polluted": 43346, + "pollution": 10384, + "polly": 31204, + "polo": 35928, + "polo": 10229, + "poly": 6833, + "poly": 18367, + "polye": 31730, + "polyester": 38514, + "polym": 23626, + "polymer": 29993, + "polyne": 38892, + "polyvore": 24771, + "pom": 7548, + "pom": 24280, + "pome": 27963, + "pomegran": 29326, + "pomegranate": 32415, + "pomer": 35156, + "pomona": 41690, + "pompe": 18352, + "pompeii": 47775, + "pompeo": 34351, + "pompey": 35079, + "pon": 3809, + "pon": 22391, + "ponce": 43637, + "pond": 10750, + "ponder": 36863, + "pondering": 47395, + "ponds": 31033, + "pone": 32183, + "pong": 40546, + "pong": 17710, + "ponies": 34157, + "pons": 41255, + "pont": 47563, + "pont": 22997, + "ponte": 40892, + "ponti": 15527, + "pontiac": 25373, + "pontifex": 33566, + "ponty": 45152, + "pony": 24438, + "pony": 12678, + "ponytail": 43265, + "poo": 6601, + "poo": 14389, + "pooch": 37037, + "poodle": 34961, + "pooh": 27103, + "pooja": 35676, + "pool": 12484, + "pool": 2831, + "poole": 26290, + "pools": 18736, + "poolside": 35509, + "poon": 33799, + "poon": 36178, + "poop": 23310, + "poor": 14528, + "poor": 3665, + "poorest": 40771, + "poorly": 21101, + "pop": 6530, + "pop": 2852, + "popart": 47425, + "popcorn": 15034, + "pope": 16994, + "pope": 9283, + "popefrancis": 37254, + "poplar": 38726, + "popo": 38835, + "popo": 35572, + "popp": 13156, + "popped": 14934, + "poppies": 30385, + "poppin": 28536, + "popping": 18152, + "poppins": 41216, + "poppy": 32194, + "poppy": 15447, + "pops": 11705, + "popsic": 38481, + "popu": 3785, + "popul": 6593, + "popular": 15854, + "popular": 4368, + "popularity": 19235, + "populated": 38420, + "population": 8423, + "populations": 23797, + "populism": 48998, + "populist": 49376, + "popup": 33053, + "por": 817, + "por": 7697, + "pora": 23537, + "porcel": 19409, + "porcelain": 20451, + "porch": 17154, + "pore": 28267, + "pork": 40379, + "pork": 7897, + "poro": 48110, + "porridge": 34924, + "porsch": 48009, + "porsche": 44049, + "porsche": 8783, + "port": 1641, + "port": 1418, + "porta": 45037, + "portable": 11949, + "portage": 32087, + "portal": 14982, + "porte": 28654, + "ported": 16879, + "porter": 28319, + "porter": 10318, + "porters": 15670, + "portfoli": 45766, + "portfolio": 11938, + "porth": 37425, + "porti": 45760, + "porting": 26052, + "portion": 13739, + "portions": 22914, + "portland": 38366, + "portland": 8880, + "portman": 34755, + "porto": 24853, + "porto": 18947, + "portobello": 48025, + "portra": 4175, + "portrait": 39312, + "portrait": 5352, + "portraits": 14203, + "portray": 46282, + "portrayal": 39238, + "portrayed": 36093, + "ports": 7734, + "portsm": 17063, + "portsmouth": 19074, + "portu": 7159, + "portugal": 9503, + "portugue": 17498, + "portuguese": 18019, + "pos": 1780, + "pos": 11839, + "pose": 25478, + "pose": 4230, + "posed": 5206, + "posei": 47270, + "poser": 46899, + "poses": 9773, + "posey": 34852, + "posh": 26748, + "posing": 10518, + "posit": 28793, + "positi": 7895, + "position": 4657, + "positioned": 34482, + "positioning": 30657, + "positions": 12188, + "positive": 21811, + "positive": 4844, + "positively": 24688, + "positivity": 19966, + "poss": 39745, + "posse": 17414, + "posse": 28413, + "possess": 36810, + "possessed": 36220, + "possession": 16154, + "possessions": 40588, + "possi": 2521, + "possibilities": 17932, + "possibility": 18517, + "possible": 3134, + "possibly": 8601, + "possum": 38575, + "post": 3489, + "post": 1549, + "postage": 27570, + "postal": 21687, + "postcard": 14785, + "postcards": 23922, + "postdoc": 41013, + "posted": 4752, + "poster": 22881, + "poster": 3574, + "posters": 9673, + "postgame": 34873, + "postgraduate": 31997, + "posthum": 42410, + "posting": 7559, + "postman": 38285, + "postpon": 23247, + "postponed": 25097, + "posts": 7824, + "postseason": 24521, + "posture": 29681, + "posure": 35539, + "pot": 3547, + "pot": 5168, + "potam": 45825, + "potassi": 36889, + "potassium": 37147, + "potat": 5975, + "potato": 8527, + "potatoes": 11567, + "potd": 28765, + "pote": 41869, + "poten": 4454, + "potent": 26082, + "potenti": 44104, + "potential": 5100, + "potentially": 16508, + "potholes": 47506, + "potion": 46055, + "potom": 38848, + "potomac": 43372, + "pots": 19234, + "pott": 28698, + "potted": 48581, + "potter": 24975, + "potter": 9026, + "pottery": 18396, + "potts": 39839, + "potty": 43569, + "potus": 8740, + "pou": 9423, + "pouch": 26811, + "poul": 22485, + "poultry": 31005, + "poun": 33719, + "pound": 33809, + "pound": 10674, + "pounding": 46544, + "pounds": 10752, + "pour": 33112, + "pour": 8180, + "poured": 26621, + "pouring": 16098, + "pours": 26005, + "pout": 39621, + "poutine": 43768, + "pov": 25731, + "pover": 8432, + "pover": 29464, + "poverty": 9095, + "pow": 1317, + "pow": 17745, + "powder": 32427, + "powder": 9674, + "powe": 36955, + "powell": 13305, + "power": 2789, + "power": 1807, + "powerball": 47803, + "powered": 45442, + "powered": 7332, + "powerful": 4875, + "powerhouse": 22858, + "powering": 16231, + "powerof": 31961, + "powerpoint": 38940, + "powerrangers": 40620, + "powers": 9422, + "pox": 43649, + "poy": 34737, + "poyn": 47655, + "poz": 39953, + "pp": 604, + "pp": 4186, + "ppa": 10416, + "ppard": 23391, + "ppc": 27778, + "ppe": 24573, + "ppe": 11867, + "pped": 1873, + "ppel": 46523, + "ppen": 30663, + "pper": 6719, + "pper": 2440, + "ppers": 5232, + "ppery": 27833, + "ppet": 20744, + "ppets": 25849, + "ppg": 27433, + "ppi": 9594, + "ppie": 33795, + "ppin": 8076, + "pping": 22214, + "pping": 1682, + "ppings": 35687, + "ppl": 6758, + "pple": 12302, + "ppm": 42053, + "ppo": 10215, + "ppor": 37613, + "ppp": 14017, + "pps": 10683, + "ppv": 38864, + "ppy": 30360, + "ppy": 3860, + "pr": 766, + "pr": 4150, + "pra": 1865, + "pra": 19285, + "prab": 17901, + "prabhas": 29959, + "prabhu": 31529, + "prac": 2243, + "practi": 29995, + "practic": 5495, + "practical": 10792, + "practically": 25588, + "practice": 3349, + "practiced": 36749, + "practices": 9040, + "practicing": 12750, + "practise": 38938, + "practising": 36478, + "practiti": 19909, + "practitioner": 32591, + "practitioners": 29045, + "prada": 29456, + "pradesh": 15384, + "prado": 44141, + "prag": 31025, + "prague": 14940, + "prairi": 12629, + "prairie": 14753, + "praise": 10013, + "praised": 27649, + "praises": 23049, + "praising": 36961, + "prakash": 43708, + "prakash": 25366, + "pram": 47774, + "pran": 20048, + "prank": 23654, + "pras": 41562, + "prasad": 29562, + "prat": 23069, + "prati": 45773, + "pratt": 37863, + "pratt": 23396, + "prawn": 33102, + "prawns": 34903, + "pray": 12671, + "pray": 6041, + "prayed": 34665, + "prayer": 41452, + "prayer": 6583, + "prayers": 8393, + "prayfor": 18443, + "praying": 11550, + "prays": 46602, + "prc": 28781, + "pre": 679, + "pre": 2900, + "preach": 22545, + "preacher": 29357, + "preaching": 23642, + "precau": 36532, + "precautions": 47845, + "prece": 15361, + "preci": 5470, + "precin": 27908, + "precinct": 32587, + "precious": 8226, + "precipit": 27463, + "precipitation": 33399, + "precise": 24457, + "precisely": 34954, + "precision": 44021, + "precision": 15621, + "pred": 40370, + "predat": 13364, + "predator": 20653, + "predators": 25569, + "prede": 38454, + "predecess": 38963, + "predic": 4876, + "predict": 16900, + "predictable": 25344, + "predicted": 18702, + "predicting": 30414, + "prediction": 16296, + "predictions": 15125, + "predictive": 29798, + "predicts": 25960, + "preds": 40125, + "pree": 47026, + "preet": 30131, + "prefe": 14542, + "prefecture": 32890, + "prefer": 33426, + "prefer": 11450, + "preference": 35057, + "preferences": 38118, + "preferred": 18772, + "prefers": 38528, + "pregame": 18575, + "pregn": 7190, + "pregnancy": 12769, + "pregnant": 11195, + "prehistoric": 32750, + "prejudice": 28337, + "preli": 15523, + "prelimin": 19990, + "preliminary": 20997, + "prelims": 43223, + "prelude": 42966, + "prem": 32090, + "prem": 21724, + "premature": 39253, + "premi": 2413, + "premier": 16996, + "premier": 5539, + "premiere": 5367, + "premiered": 27652, + "premieres": 19907, + "premiering": 32615, + "premierleague": 22608, + "premiers": 44883, + "premiership": 23665, + "premiosm": 38460, + "premiosmtvmiaw": 38630, + "premise": 45952, + "premises": 27266, + "premium": 8011, + "pren": 20801, + "preneur": 46288, + "preorder": 16703, + "preorders": 45985, + "prep": 6430, + "prep": 7277, + "prepa": 26270, + "prepaid": 42934, + "prepar": 4968, + "preparation": 11651, + "preparations": 19135, + "prepare": 7014, + "prepared": 7677, + "preparedness": 29492, + "prepares": 16375, + "preparing": 7365, + "prepped": 34379, + "prepping": 16459, + "preps": 14765, + "prequel": 40461, + "pres": 1385, + "pres": 8529, + "presale": 27135, + "presby": 30447, + "presbyter": 33959, + "presbyterian": 35370, + "preschool": 24354, + "prescott": 29392, + "prescri": 14851, + "prescribed": 36968, + "prescription": 23061, + "preseason": 13813, + "presen": 16742, + "presence": 8848, + "present": 2344, + "present": 2881, + "presentation": 4594, + "presentations": 16998, + "presented": 4587, + "presenter": 18587, + "presenters": 32759, + "presenting": 5339, + "presents": 4215, + "preserv": 17616, + "preservation": 21074, + "preserve": 15570, + "preserved": 23161, + "preserves": 44881, + "preserving": 32315, + "presi": 1697, + "presiden": 43374, + "presidency": 18077, + "president": 19900, + "president": 1940, + "presidente": 47363, + "presidenti": 48297, + "presidential": 8503, + "presidents": 16726, + "presiding": 45298, + "presley": 30013, + "press": 4124, + "press": 2124, + "pressed": 20080, + "presser": 27826, + "presses": 33748, + "pressing": 20893, + "pressure": 6083, + "pressures": 38487, + "prest": 41840, + "presti": 12245, + "prestige": 29328, + "prestigious": 15888, + "presto": 42211, + "preston": 37335, + "preston": 15179, + "presu": 21667, + "presumably": 42562, + "pret": 9652, + "preten": 15871, + "pretend": 18111, + "pretending": 21306, + "pretoria": 36080, + "prett": 46667, + "prettier": 31745, + "prettiest": 22866, + "pretty": 18286, + "pretty": 2111, + "pretz": 24890, + "pretzel": 36707, + "pretzels": 45468, + "prev": 20274, + "prevail": 31637, + "prevalence": 41729, + "prevalent": 46260, + "preven": 29382, + "prevent": 26436, + "prevent": 7968, + "preventable": 44250, + "prevented": 35356, + "preventing": 21756, + "prevention": 9500, + "preventive": 40949, + "prevents": 31746, + "preview": 4449, + "previews": 20279, + "previous": 9252, + "previously": 13359, + "prey": 17131, + "prez": 17956, + "pri": 955, + "pri": 23400, + "pric": 24275, + "price": 13254, + "price": 2827, + "priced": 16934, + "priceless": 15743, + "prices": 5954, + "pricing": 14800, + "prick": 43921, + "prick": 46516, + "pride": 15323, + "pride": 3436, + "pridemonth": 41410, + "prie": 22477, + "priest": 38756, + "priest": 14222, + "priests": 30005, + "prim": 22004, + "prima": 35611, + "prima": 33277, + "primal": 36604, + "primar": 21579, + "primaries": 46126, + "primarily": 29465, + "primark": 48329, + "primary": 35024, + "primary": 5814, + "primavera": 44899, + "prime": 14162, + "prime": 5183, + "primed": 45694, + "primer": 22388, + "primetime": 29763, + "primitive": 37467, + "primo": 43215, + "primrose": 45891, + "prin": 1588, + "prince": 9457, + "prince": 4735, + "princes": 45329, + "princes": 30136, + "princess": 24123, + "princess": 5079, + "princesses": 34161, + "princeton": 22433, + "princi": 5129, + "principal": 33599, + "principal": 8860, + "principals": 27524, + "principle": 19595, + "principles": 13755, + "print": 17851, + "print": 3557, + "printable": 29648, + "printed": 7978, + "printer": 14521, + "printers": 27881, + "printing": 7369, + "printmaking": 38669, + "prints": 7704, + "prior": 20328, + "prior": 10572, + "priorit": 47773, + "prioriti": 28822, + "priorities": 15232, + "prioritize": 46715, + "priority": 12451, + "priory": 38665, + "prisc": 32468, + "priscilla": 42396, + "prise": 23343, + "prism": 49311, + "prism": 34356, + "prison": 9281, + "prison": 6622, + "prisoner": 21427, + "prisoners": 17460, + "prisons": 26607, + "pristine": 30618, + "prit": 41668, + "prit": 37523, + "prith": 39173, + "prius": 43561, + "priv": 3270, + "privacy": 10437, + "private": 20362, + "private": 4439, + "privately": 32970, + "privati": 27379, + "privi": 8367, + "privileg": 18015, + "privilege": 11537, + "privileged": 18166, + "prix": 10875, + "priya": 31275, + "priyan": 16488, + "priyanka": 31959, + "priyankach": 30030, + "priyankachopra": 30264, + "prize": 48222, + "prize": 4521, + "prized": 38769, + "prizes": 9268, + "prk": 37094, + "pro": 644, + "pro": 2630, + "proactive": 33364, + "prob": 17706, + "prob": 24007, + "probab": 3907, + "probability": 32637, + "probable": 42444, + "probably": 4047, + "probation": 36531, + "probe": 14359, + "probes": 48564, + "probiotics": 49395, + "proble": 2719, + "problem": 4324, + "problematic": 33767, + "problems": 4671, + "probs": 16330, + "probz": 34243, + "proc": 38417, + "proce": 4076, + "procedu": 18204, + "procedural": 48177, + "procedure": 20163, + "procedures": 21109, + "proceed": 26664, + "proceed": 33894, + "proceedings": 26953, + "proceeds": 11882, + "process": 17291, + "process": 4078, + "processed": 23816, + "processes": 15169, + "processing": 11737, + "procession": 26288, + "processor": 22838, + "processors": 43634, + "proclaimed": 34489, + "proclamation": 32065, + "procra": 25361, + "procrastin": 25586, + "procrastination": 42825, + "procreate": 39336, + "proctor": 47204, + "procu": 21001, + "procurement": 23733, + "prod": 44349, + "prod": 11991, + "prodi": 27759, + "prodigy": 31973, + "produ": 27852, + "produc": 1471, + "produce": 7529, + "produced": 7479, + "producer": 7064, + "producers": 13883, + "produces": 19940, + "producing": 13579, + "product": 32602, + "product": 4306, + "production": 4146, + "productions": 14166, + "productive": 9697, + "productivity": 12800, + "products": 3964, + "prof": 15043, + "prof": 5488, + "profe": 2611, + "profess": 5486, + "professi": 3705, + "profession": 8104, + "profession": 19671, + "professional": 46007, + "professional": 4774, + "professionalism": 41252, + "professionally": 33892, + "professionals": 10165, + "professor": 47302, + "professor": 6092, + "professors": 27758, + "profici": 34685, + "profile": 14291, + "profile": 6444, + "profiles": 22070, + "profiling": 37123, + "profit": 16941, + "profit": 7909, + "profitable": 25465, + "profits": 13410, + "profound": 48245, + "profound": 22998, + "profs": 19260, + "prog": 22219, + "progno": 46070, + "program": 4162, + "program": 2737, + "programme": 6322, + "programmer": 37001, + "programmes": 20468, + "programming": 10831, + "programs": 7345, + "progre": 7069, + "progress": 4421, + "progressi": 23297, + "progressing": 32346, + "progression": 24772, + "progressive": 12208, + "progressives": 41709, + "prohi": 41124, + "prohib": 45040, + "prohibition": 34440, + "proj": 39156, + "proje": 48345, + "projec": 1610, + "project": 15911, + "project": 1965, + "projected": 22873, + "projection": 22384, + "projections": 34638, + "projector": 27816, + "projects": 5090, + "proli": 19710, + "prolife": 32126, + "prolifer": 39018, + "prolific": 27839, + "prolly": 45968, + "prolon": 35379, + "prolonged": 41972, + "prom": 40363, + "prom": 7944, + "prome": 34355, + "promen": 33578, + "promenade": 35522, + "promethe": 44183, + "promin": 35217, + "prominent": 19172, + "promis": 3963, + "promise": 6745, + "promised": 11516, + "promises": 12064, + "promising": 14183, + "promo": 3037, + "promo": 6755, + "promos": 35044, + "promote": 47384, + "promote": 8003, + "promoted": 16395, + "promoter": 33081, + "promotes": 20169, + "promoting": 9695, + "promotion": 9259, + "promotional": 17619, + "promotions": 19142, + "promp": 11671, + "prompt": 20198, + "prompted": 45746, + "prompts": 33490, + "proms": 37759, + "pron": 13285, + "prone": 30964, + "pronoun": 23022, + "pronounce": 40489, + "pronounced": 34109, + "pronto": 44296, + "proof": 17020, + "proof": 5248, + "proofing": 35679, + "proofs": 41023, + "prop": 19123, + "prop": 16254, + "propag": 12151, + "propaganda": 14718, + "propane": 45546, + "propel": 48439, + "propeller": 47404, + "proper": 3577, + "proper": 8205, + "properly": 12560, + "properties": 10922, + "property": 26486, + "property": 5043, + "prophe": 9662, + "prophecy": 32501, + "prophet": 15549, + "prophetic": 47476, + "prophets": 39441, + "propor": 35016, + "proportion": 35775, + "proportions": 39391, + "propos": 9455, + "proposal": 12139, + "proposals": 20568, + "propose": 28471, + "proposed": 10615, + "proposes": 27133, + "proposing": 42631, + "proposition": 44780, + "propri": 28243, + "props": 15249, + "propulsion": 49380, + "pros": 33925, + "pros": 14147, + "prosciutto": 46565, + "prose": 47063, + "prose": 28675, + "prosecco": 28839, + "prosecu": 12136, + "prosecution": 30902, + "prosecutor": 23736, + "prosecutors": 31656, + "prosp": 24242, + "prospec": 12693, + "prospect": 11211, + "prospective": 28034, + "prospects": 15372, + "prosper": 16121, + "prosper": 33526, + "prosperity": 17203, + "prosperous": 28252, + "prost": 47923, + "prostate": 28808, + "prostatec": 49064, + "prosthetic": 44602, + "prostitu": 37333, + "protag": 28950, + "protagonist": 38183, + "prote": 1845, + "protec": 5640, + "protect": 25563, + "protect": 4817, + "protected": 12266, + "protecting": 11710, + "protection": 6238, + "protections": 33772, + "protective": 17028, + "protector": 20441, + "protectors": 45039, + "protects": 21889, + "protein": 8088, + "proteins": 28661, + "protest": 6279, + "protestant": 46945, + "protested": 48089, + "protester": 42073, + "protesters": 12660, + "protesting": 18788, + "protestors": 27822, + "protests": 12450, + "proto": 8672, + "proto": 44958, + "protocol": 19938, + "protocols": 39631, + "proton": 40009, + "prototype": 16675, + "prototyping": 42081, + "prou": 5739, + "proud": 11080, + "proud": 1679, + "prouder": 39585, + "proudest": 46806, + "proudly": 11203, + "proudof": 48184, + "proudtobe": 35043, + "prov": 23772, + "prov": 35021, + "prove": 10107, + "proved": 16473, + "proven": 35405, + "proven": 14569, + "provence": 28067, + "prover": 18312, + "proverb": 34419, + "proverbs": 27016, + "proves": 16119, + "provi": 2289, + "provide": 4832, + "provided": 9046, + "providence": 19331, + "provider": 14409, + "providers": 17120, + "provides": 7161, + "providing": 7250, + "provin": 12074, + "province": 8978, + "provinces": 35050, + "provincial": 16002, + "proving": 18055, + "provision": 30148, + "provisional": 36008, + "provisions": 39269, + "provo": 15367, + "provoc": 31618, + "provocative": 43809, + "provoking": 25510, + "provost": 36627, + "prow": 38737, + "prowrestling": 39825, + "prox": 41616, + "proxim": 31436, + "proximity": 38298, + "proxy": 31680, + "prs": 23879, + "pru": 12961, + "pruitt": 39453, + "prun": 29029, + "pruning": 48133, + "pry": 31965, + "pryor": 43375, + "ps": 3982, + "ps": 814, + "psa": 14031, + "psal": 13859, + "psalm": 17995, + "psalms": 35003, + "psb": 37017, + "psc": 43118, + "psd": 28810, + "pse": 19737, + "pse": 5423, + "pseu": 24919, + "pseudo": 46618, + "psg": 17123, + "psi": 45848, + "psi": 24533, + "psic": 29299, + "psis": 33041, + "psl": 21373, + "psn": 36781, + "pso": 27045, + "pson": 7487, + "psori": 44688, + "psp": 32769, + "pss": 35718, + "pss": 42535, + "psst": 47814, + "pst": 12692, + "psu": 41286, + "psu": 28338, + "psv": 44530, + "psy": 3576, + "psy": 11056, + "psych": 31041, + "psych": 20509, + "psyched": 19932, + "psyched": 35199, + "psychedelic": 23292, + "psychi": 18147, + "psychiatric": 30578, + "psychiatry": 39706, + "psychic": 24916, + "psycho": 6472, + "psycho": 22154, + "psychological": 18153, + "psychologist": 32827, + "psychology": 12352, + "psychop": 30112, + "psychotic": 48774, + "pt": 11139, + "pt": 1459, + "pta": 11586, + "ptbo": 40481, + "ptc": 44646, + "pte": 47804, + "pter": 49323, + "pti": 29375, + "pti": 10491, + "ptic": 20670, + "ption": 3479, + "ptions": 24963, + "pto": 31372, + "pto": 34092, + "pton": 19780, + "pts": 5886, + "ptsd": 23973, + "ptv": 42402, + "pu": 755, + "pu": 11780, + "pub": 20720, + "pub": 6301, + "puberty": 44122, + "pubg": 31496, + "publ": 3434, + "publi": 1617, + "public": 3592, + "public": 2122, + "publica": 49007, + "publication": 13538, + "publications": 27334, + "publichealth": 35872, + "publicity": 20831, + "publicly": 18554, + "publish": 19032, + "published": 4311, + "publisher": 20455, + "publishers": 25222, + "publishes": 35633, + "publishing": 10994, + "publix": 47985, + "pubs": 21099, + "puc": 48779, + "puck": 17550, + "pud": 39234, + "pudding": 14025, + "puddle": 33545, + "pue": 20161, + "pueblo": 33076, + "puer": 8968, + "puerto": 12289, + "puertor": 22757, + "puertorico": 26356, + "puff": 44477, + "puff": 17184, + "puffin": 47632, + "puffs": 47453, + "puffy": 49245, + "pug": 20950, + "pug": 17739, + "pugchat": 42266, + "pugh": 41302, + "puglia": 38345, + "pugs": 39425, + "puj": 46163, + "puja": 33753, + "puk": 31811, + "pul": 2469, + "pul": 40512, + "pula": 45856, + "puli": 47293, + "pulit": 27745, + "pulitzer": 31419, + "pull": 20155, + "pull": 6857, + "pulled": 8525, + "pulling": 12897, + "pullman": 40203, + "pullover": 44020, + "pulls": 16041, + "pulmon": 32613, + "pulmonary": 39132, + "pulp": 25410, + "pulse": 40091, + "pulse": 12485, + "pulses": 42177, + "pulsion": 35398, + "pum": 37497, + "puma": 20858, + "pump": 5179, + "pump": 9173, + "pumped": 12796, + "pumping": 25150, + "pumpkin": 36386, + "pumpkin": 8842, + "pumpkins": 23787, + "pumps": 18540, + "pun": 2707, + "pun": 19929, + "punc": 43907, + "punch": 29332, + "punch": 10730, + "punched": 31689, + "punches": 35279, + "punching": 33468, + "punctu": 31565, + "punctuation": 47051, + "pundit": 41466, + "pune": 32593, + "pune": 14488, + "pung": 45420, + "puni": 11479, + "punish": 34569, + "punished": 31598, + "punisher": 38509, + "punishment": 19099, + "punjab": 19405, + "punjab": 12883, + "punjabi": 25430, + "punk": 28933, + "punk": 7246, + "punks": 47171, + "puns": 35231, + "punt": 32699, + "punta": 34112, + "punter": 47092, + "pup": 11926, + "pup": 11302, + "pupil": 27265, + "pupils": 13628, + "pupp": 7116, + "puppet": 18439, + "puppets": 28475, + "puppies": 14820, + "puppy": 25431, + "puppy": 6829, + "puppylove": 40849, + "pups": 20778, + "pur": 1727, + "pur": 6265, + "pura": 25596, + "puram": 46174, + "purcell": 46065, + "purch": 8384, + "purchase": 5481, + "purchased": 13399, + "purchases": 21887, + "purchasing": 20718, + "purdu": 40691, + "purdue": 22280, + "pure": 14202, + "pure": 5979, + "puree": 45474, + "purely": 32459, + "puremichigan": 39783, + "purest": 45497, + "purge": 33514, + "puri": 16910, + "puri": 21974, + "purification": 47724, + "purity": 29780, + "purple": 17837, + "purple": 5496, + "purpose": 33492, + "purpose": 7391, + "purposes": 22020, + "purr": 49262, + "purr": 46343, + "purse": 16480, + "pursue": 19463, + "pursuing": 26424, + "pursuit": 16469, + "purée": 40981, + "pus": 13841, + "pusa": 40825, + "push": 16028, + "push": 6831, + "pushaw": 35407, + "pushaward": 35448, + "pushawards": 47184, + "pushed": 16155, + "pushes": 23828, + "pushing": 11549, + "put": 29535, + "put": 1983, + "putin": 10693, + "putnam": 40235, + "puts": 7898, + "putt": 30279, + "putter": 44723, + "putting": 5154, + "puzz": 19760, + "puzzle": 12875, + "puzzles": 27986, + "pv": 14517, + "pv": 13495, + "pvc": 26959, + "pvp": 44172, + "pvt": 29898, + "pw": 19419, + "pw": 16067, + "pwc": 22965, + "px": 24790, + "px": 10262, + "pxrtg": 36262, + "py": 4005, + "py": 7504, + "pye": 31099, + "pyeongchang": 36066, + "pyg": 41450, + "pyram": 14405, + "pyramid": 18725, + "pyramids": 36877, + "pyrene": 36740, + "pyrenees": 39744, + "pyro": 39762, + "python": 13370, + "pz": 48361, + "pé": 43167, + "q": 80, + "q": 336, + "qa": 24944, + "qa": 16360, + "qad": 27844, + "qadri": 35672, + "qaeda": 31246, + "qanda": 48672, + "qanon": 19182, + "qant": 35404, + "qantas": 43250, + "qatar": 32804, + "qatar": 10872, + "qb": 8073, + "qbs": 38188, + "qc": 17406, + "qe": 30974, + "qf": 27215, + "qi": 25054, + "qi": 11256, + "qing": 46522, + "qing": 34339, + "ql": 28366, + "qld": 23039, + "qld": 13765, + "qldpol": 42296, + "qm": 42148, + "qotd": 24504, + "qpr": 24788, + "qq": 31960, + "qr": 18193, + "qs": 14364, + "qt": 15013, + "qtr": 44803, + "qu": 666, + "qu": 28646, + "qua": 20363, + "quack": 45575, + "quad": 11656, + "quad": 13419, + "quadcopter": 39792, + "quadru": 35831, + "quaid": 34265, + "quail": 34392, + "quaint": 45976, + "quake": 8421, + "quaker": 43395, + "quakes": 24572, + "qual": 9979, + "qual": 32405, + "qualcomm": 38683, + "quali": 4574, + "qualification": 21508, + "qualifications": 35225, + "qualified": 11927, + "qualifier": 18733, + "qualifiers": 21388, + "qualifies": 35820, + "qualify": 17019, + "qualifying": 11895, + "qualitative": 45847, + "qualities": 20488, + "quality": 28545, + "quality": 3027, + "quan": 11669, + "quan": 27490, + "quand": 28198, + "quant": 15050, + "quanti": 31540, + "quantitative": 40583, + "quantities": 33917, + "quantity": 26920, + "quantum": 15320, + "quar": 3856, + "quare": 42549, + "quarry": 27601, + "quart": 7851, + "quarter": 8816, + "quarter": 6632, + "quarterback": 16545, + "quarterfinal": 37992, + "quarterfinals": 28971, + "quarterly": 23350, + "quarters": 10146, + "quartet": 18056, + "quartz": 17752, + "quat": 25715, + "quattro": 40300, + "quay": 40276, + "quay": 17304, + "que": 1147, + "que": 2319, + "quebec": 15373, + "queen": 6407, + "queen": 2997, + "queenof": 44398, + "queens": 22943, + "queens": 9330, + "queensland": 15168, + "queer": 38874, + "queer": 18161, + "quel": 39774, + "quel": 21879, + "quen": 23876, + "quen": 38324, + "quent": 23808, + "quentin": 27530, + "quer": 17378, + "quer": 26859, + "quered": 23210, + "queries": 32958, + "querque": 30338, + "query": 27464, + "ques": 25328, + "ques": 7715, + "queso": 40110, + "quest": 31653, + "quest": 4846, + "questi": 2391, + "question": 18961, + "question": 4382, + "questionable": 30733, + "questioned": 31847, + "questioning": 24887, + "questions": 3883, + "quests": 44611, + "quet": 8513, + "quets": 39055, + "quetta": 38326, + "quette": 18993, + "queu": 32705, + "queue": 18549, + "queues": 40649, + "queuing": 44082, + "quez": 18677, + "quezon": 41117, + "qui": 1912, + "qui": 18046, + "quic": 26474, + "quiche": 47723, + "quick": 5969, + "quick": 3712, + "quicker": 29211, + "quickest": 37734, + "quickly": 7787, + "quid": 30732, + "quie": 43875, + "quien": 43482, + "quiere": 42723, + "quiero": 32567, + "quiet": 17853, + "quiet": 7557, + "quietly": 22208, + "quig": 44690, + "quil": 12305, + "quill": 48951, + "quilt": 23977, + "quilted": 46052, + "quin": 8607, + "quin": 17167, + "quincy": 27640, + "quind": 32339, + "quinn": 12306, + "quinoa": 26703, + "quins": 39701, + "quint": 26898, + "quinta": 47446, + "quinte": 22098, + "quintess": 37538, + "quintet": 35125, + "quipment": 42813, + "quir": 15943, + "quirky": 25044, + "quis": 15064, + "quist": 25128, + "quit": 19358, + "quit": 11140, + "quite": 4135, + "quito": 35828, + "quits": 32505, + "quitting": 33871, + "quity": 33133, + "quiz": 31197, + "quiz": 8344, + "quizz": 35041, + "quo": 3046, + "quo": 28127, + "quoi": 45549, + "quot": 5452, + "quot": 47587, + "quota": 42097, + "quotation": 49195, + "quote": 15446, + "quote": 4020, + "quoted": 27706, + "quoteoftheday": 19975, + "quotes": 5808, + "quoting": 31651, + "qur": 37782, + "quran": 19690, + "qureshi": 46307, + "qvist": 42322, + "qx": 45038, + "r": 81, + "r": 337, + "ra": 559, + "ra": 1735, + "raa": 44344, + "rab": 14816, + "rab": 33224, + "rabb": 6875, + "rabbi": 20959, + "rabbit": 10274, + "rabbits": 27028, + "rabhu": 25806, + "rable": 10182, + "rac": 1773, + "rac": 30462, + "raccoon": 29516, + "race": 10978, + "race": 2471, + "racec": 18814, + "racecourse": 25036, + "raced": 36021, + "racer": 16798, + "racers": 33603, + "races": 8605, + "raceway": 24650, + "rach": 6876, + "rach": 33429, + "racha": 21952, + "racha": 35022, + "rachael": 29095, + "rachel": 13511, + "rachel": 8029, + "raci": 33381, + "racial": 13801, + "racially": 43577, + "racing": 23306, + "racing": 3699, + "racism": 11276, + "racist": 9684, + "racists": 41777, + "rack": 24600, + "rack": 12034, + "racket": 37691, + "racks": 21191, + "rad": 4473, + "rad": 8238, + "rada": 30437, + "radar": 9672, + "radcliffe": 33096, + "rade": 44494, + "rade": 17911, + "rader": 45002, + "radford": 45800, + "radha": 43122, + "radi": 5772, + "radial": 42028, + "radiance": 45670, + "radiant": 25614, + "radiation": 18210, + "radiator": 39372, + "radic": 18082, + "radical": 13712, + "radicals": 45903, + "radio": 7176, + "radio": 2638, + "radioactive": 34704, + "radiodisney": 36483, + "radiohead": 39472, + "radiology": 29684, + "radios": 43669, + "radish": 37789, + "radius": 37570, + "rado": 29784, + "rae": 21646, + "rae": 15051, + "rael": 45390, + "raer": 44561, + "raf": 11495, + "raf": 11490, + "rafa": 14352, + "rafa": 24850, + "rafael": 38221, + "rafael": 19216, + "rafaelnadal": 49219, + "raff": 34900, + "raffic": 32928, + "raffle": 13752, + "raffles": 43489, + "rafi": 35304, + "raft": 9233, + "rafting": 36309, + "rag": 13958, + "rag": 20687, + "rage": 8593, + "rages": 34253, + "ragh": 35642, + "ragha": 40972, + "raging": 25015, + "ragn": 24125, + "ragnar": 34385, + "ragnarok": 41856, + "ragon": 34768, + "rags": 47838, + "rah": 12277, + "rah": 8766, + "raheem": 43317, + "rahim": 24152, + "rahman": 19680, + "rahu": 13129, + "rahul": 37239, + "rahul": 17440, + "rahulg": 27510, + "rahulgandhi": 28293, + "rai": 9165, + "rai": 9638, + "raid": 6877, + "raided": 43417, + "raider": 27368, + "raider": 21455, + "raidernation": 47901, + "raiders": 11817, + "raids": 26655, + "rail": 4573, + "rail": 6879, + "raila": 47273, + "railminindia": 35557, + "railroad": 17080, + "rails": 23427, + "railway": 27614, + "railway": 7856, + "railwayana": 46750, + "railways": 20765, + "raim": 45785, + "rain": 3128, + "rain": 2443, + "raina": 30564, + "rainbow": 24562, + "rainbow": 6286, + "rainbows": 30483, + "raine": 49038, + "raine": 6871, + "rained": 32310, + "rainf": 15024, + "rainfall": 15350, + "rainforest": 22823, + "rainier": 37850, + "raining": 13964, + "rains": 14272, + "rainy": 10222, + "rais": 14729, + "raise": 24249, + "raise": 5078, + "raised": 6027, + "raiser": 33555, + "raises": 13297, + "raisethe": 47109, + "raisin": 36864, + "raising": 6883, + "raj": 5958, + "raj": 10813, + "raja": 46069, + "raja": 19150, + "rajan": 46595, + "rajas": 16185, + "rajasthan": 18017, + "raje": 21899, + "rajesh": 43602, + "raji": 27569, + "rajini": 29600, + "rajini": 40622, + "rajinikanth": 32922, + "rajiv": 40197, + "rajkumar": 49304, + "rajput": 47572, + "raju": 47029, + "rak": 13523, + "rak": 26287, + "rake": 26825, + "rake": 32712, + "rakesh": 41083, + "ral": 8062, + "ral": 1406, + "rale": 14192, + "raleigh": 18207, + "rall": 23249, + "rallies": 25230, + "rally": 18882, + "rally": 5041, + "rallying": 36836, + "ralph": 25290, + "ralph": 12234, + "ram": 1976, + "ram": 2007, + "rama": 22112, + "ramad": 12736, + "ramadan": 15547, + "ramadhan": 47415, + "raman": 39816, + "ramapho": 43963, + "ramaphosa": 44993, + "ramatta": 49112, + "rambo": 41855, + "ramcharan": 45275, + "rame": 47745, + "ramen": 18892, + "ramesh": 48640, + "ramesh": 40186, + "rami": 43016, + "ramirez": 23877, + "ramon": 27958, + "ramone": 47201, + "ramos": 21046, + "ramp": 14271, + "rampage": 32077, + "rampant": 41985, + "ramps": 35257, + "rams": 10292, + "ramsay": 26259, + "ramsey": 19215, + "ran": 1433, + "ran": 4031, + "rana": 22143, + "ranbir": 40881, + "rance": 29034, + "ranch": 43955, + "ranch": 10659, + "rancho": 26258, + "rand": 5628, + "rand": 18718, + "randall": 23639, + "rande": 21469, + "randolph": 29899, + "random": 11396, + "random": 6160, + "randomly": 17272, + "rands": 39153, + "randy": 29479, + "randy": 13279, + "rane": 28852, + "rang": 4043, + "rang": 24377, + "range": 13627, + "range": 3818, + "ranger": 31472, + "ranger": 13593, + "rangers": 7664, + "ranges": 25685, + "ranging": 25946, + "rani": 29264, + "rani": 22631, + "rank": 11501, + "ranked": 8307, + "rankin": 37539, + "ranking": 12347, + "rankings": 12596, + "ranks": 14469, + "rano": 18608, + "rans": 46259, + "ransom": 28523, + "ransom": 34646, + "ransomware": 33815, + "rant": 46467, + "rant": 9819, + "rants": 34014, + "ranveer": 32402, + "ranveer": 41482, + "ranveerofficial": 42116, + "rao": 16913, + "rap": 7773, + "rap": 7348, + "rape": 46099, + "rape": 10070, + "raped": 23700, + "rapha": 22754, + "raphael": 30091, + "rapi": 8610, + "rapid": 47697, + "rapid": 12205, + "rapidly": 16710, + "rapids": 18848, + "raping": 44926, + "rapist": 33360, + "rapp": 19283, + "rapper": 11860, + "rappers": 30315, + "rapping": 42864, + "raps": 37887, + "raptor": 26762, + "raptors": 17035, + "raq": 39787, + "raq": 43312, + "raqqa": 47074, + "raquel": 44338, + "rar": 26819, + "rar": 24605, + "rard": 21012, + "rare": 18992, + "rare": 3865, + "rarely": 17315, + "rarest": 43237, + "rarity": 45862, + "ras": 23492, + "ras": 8224, + "rasc": 30085, + "rascal": 43481, + "rash": 14917, + "rash": 30608, + "rashad": 46527, + "rasheed": 41638, + "rashi": 19426, + "rashid": 26757, + "rasp": 10487, + "raspberries": 37742, + "raspberry": 40162, + "raspberry": 13615, + "raspberrypi": 43934, + "rass": 45654, + "rasta": 47002, + "rat": 3806, + "rat": 8985, + "rata": 28568, + "ratchet": 25078, + "rate": 5068, + "rated": 8183, + "rates": 6864, + "rath": 18268, + "rath": 39772, + "rather": 5252, + "rati": 11486, + "rating": 10567, + "ratings": 14176, + "ratio": 15893, + "ration": 27002, + "ration": 35662, + "rational": 33086, + "ratna": 49078, + "ratri": 32288, + "rats": 19043, + "ratt": 20737, + "ratt": 34785, + "rattle": 40824, + "rattle": 41839, + "rau": 27744, + "raul": 30218, + "raun": 41169, + "rav": 14367, + "rav": 23606, + "rave": 38784, + "rave": 17601, + "ravel": 27927, + "raven": 10269, + "raven": 16803, + "ravens": 17946, + "ravi": 22947, + "ravi": 19538, + "ravin": 39099, + "raving": 45807, + "raviol": 41104, + "ravioli": 43460, + "raw": 10166, + "raw": 6323, + "rawlings": 40662, + "rax": 38520, + "ray": 5312, + "ray": 3077, + "raya": 29991, + "raymond": 16683, + "rayn": 47852, + "rayon": 47900, + "rays": 11064, + "raz": 9700, + "raz": 19087, + "raza": 37724, + "razer": 33832, + "razor": 24934, + "razor": 21300, + "razz": 43769, + "rb": 12740, + "rb": 7477, + "rbc": 37500, + "rbi": 15687, + "rbs": 29102, + "rc": 7575, + "rc": 7457, + "rca": 33942, + "rcb": 45240, + "rcmp": 31489, + "rcn": 49370, + "rctid": 49223, + "rd": 13501, + "rd": 1973, + "rda": 45755, + "rdr": 44364, + "rds": 32378, + "re": 515, + "re": 810, + "rea": 11521, + "reach": 4483, + "reach": 4279, + "reached": 6878, + "reaches": 14462, + "reaching": 11358, + "react": 36566, + "react": 15065, + "reacted": 42515, + "reacting": 40595, + "reaction": 7189, + "reactions": 18438, + "reactive": 42072, + "reactjs": 46173, + "reactor": 32037, + "reacts": 23115, + "read": 933, + "read": 1199, + "reader": 9884, + "readers": 10335, + "readiness": 28131, + "reading": 17556, + "reading": 2337, + "readingfc": 47428, + "readings": 23361, + "reads": 6597, + "ready": 17351, + "ready": 1112, + "reagan": 17767, + "real": 2017, + "real": 1532, + "realdonaldtrump": 7025, + "reale": 5930, + "realest": 45855, + "realestate": 32937, + "realestate": 6569, + "reali": 4185, + "realis": 38114, + "realise": 14773, + "realised": 17945, + "realising": 39537, + "realism": 20024, + "realist": 30248, + "realistic": 16157, + "realities": 32443, + "reality": 46802, + "reality": 5004, + "realization": 40402, + "realize": 7538, + "realized": 10489, + "realizes": 42918, + "realizing": 23284, + "reall": 39686, + "really": 43249, + "really": 1414, + "realm": 23083, + "realmadrid": 27866, + "realms": 43033, + "realness": 46761, + "realtime": 44002, + "realtime": 38203, + "realtor": 18038, + "realtors": 31759, + "realty": 20471, + "ream": 37242, + "ream": 15219, + "rean": 48477, + "reap": 31334, + "reaper": 29922, + "rear": 39652, + "rear": 10223, + "reas": 9121, + "reason": 12882, + "reason": 3893, + "reasonable": 18558, + "reasonably": 38589, + "reasoning": 30341, + "reasons": 5686, + "reau": 32398, + "reb": 12370, + "reb": 18796, + "reba": 48543, + "rebate": 43817, + "rebe": 25227, + "rebec": 10774, + "rebecca": 12892, + "rebel": 8185, + "rebel": 12248, + "rebellion": 22170, + "rebels": 13623, + "rebirth": 33303, + "reboot": 22385, + "reborn": 30229, + "reboun": 43381, + "rebound": 31280, + "rebounds": 19190, + "rebs": 28164, + "rebu": 43162, + "rebuild": 20022, + "rebuilding": 30880, + "rebuilt": 33137, + "rec": 1020, + "rec": 11243, + "recall": 15151, + "recalled": 32142, + "recalling": 47855, + "recalls": 24740, + "recap": 29816, + "recap": 8337, + "recaps": 47997, + "recard": 35536, + "rece": 1890, + "recei": 2148, + "receip": 38503, + "receipt": 30479, + "receipts": 41181, + "receive": 4800, + "received": 4178, + "receiver": 17659, + "receivers": 45294, + "receives": 10027, + "receiving": 7252, + "recent": 3969, + "recently": 4482, + "recep": 17450, + "reception": 8364, + "receptions": 46881, + "receptor": 41835, + "recess": 38182, + "recession": 27176, + "recharge": 29396, + "rechargeable": 37516, + "reci": 2037, + "recipe": 28923, + "recipe": 4614, + "recipeoftheday": 38727, + "recipes": 9243, + "recipi": 10136, + "recipient": 13703, + "recipients": 18940, + "recipro": 41789, + "recital": 23457, + "recite": 48824, + "reck": 11715, + "reckless": 26284, + "reckon": 23854, + "recl": 42277, + "reclaim": 35969, + "reclaimed": 32648, + "reco": 2535, + "reco": 46038, + "recogn": 6343, + "recogni": 5329, + "recognise": 19824, + "recognised": 20986, + "recognising": 48423, + "recognition": 9415, + "recognizable": 47240, + "recognize": 10905, + "recognized": 9929, + "recognizes": 26909, + "recognizing": 19666, + "recomm": 4540, + "recommend": 11628, + "recommend": 8942, + "recommendation": 20118, + "recommendations": 16516, + "recommended": 11100, + "recommending": 44301, + "recommends": 22940, + "recon": 15371, + "recon": 28996, + "reconciliation": 26451, + "reconstruction": 24955, + "recor": 1723, + "record": 21328, + "record": 2717, + "recorded": 9392, + "recorder": 26747, + "recording": 48237, + "recording": 6942, + "recordings": 19715, + "records": 4529, + "recover": 16785, + "recovered": 16444, + "recovering": 19005, + "recovers": 47935, + "recovery": 6591, + "recre": 22148, + "recreate": 29775, + "recreated": 40888, + "recreating": 48224, + "recreation": 17331, + "recreational": 24329, + "recru": 4745, + "recruit": 9011, + "recruit": 15585, + "recruited": 36518, + "recruiter": 43120, + "recruiters": 46542, + "recruiting": 10533, + "recruitment": 10541, + "recruits": 22647, + "recs": 33069, + "rectan": 43041, + "rectangular": 43321, + "rector": 41585, + "recu": 26798, + "recur": 19983, + "recurring": 35912, + "recy": 6790, + "recycla": 40659, + "recyclable": 48907, + "recycle": 19366, + "recycled": 16829, + "recycling": 12566, + "red": 1893, + "red": 736, + "redbubble": 46137, + "redbull": 29483, + "redbull": 29219, + "redcarpet": 32259, + "redcross": 30659, + "redd": 22149, + "redd": 40618, + "redding": 41061, + "reddish": 43383, + "reddit": 15226, + "reddy": 23028, + "rede": 10913, + "redeem": 37449, + "redefining": 46352, + "redemption": 20233, + "redesign": 24188, + "redesigned": 33111, + "redevelopment": 30322, + "redhead": 36267, + "redi": 7976, + "redman": 44753, + "redmond": 39627, + "rednation": 28180, + "rednationrising": 28262, + "redneck": 39105, + "redness": 22626, + "redo": 42524, + "redon": 48506, + "redro": 37722, + "reds": 11221, + "redskins": 19023, + "redsox": 19144, + "reduc": 5015, + "reduce": 6604, + "reduced": 10821, + "reduces": 20539, + "reducing": 13836, + "reduction": 12219, + "reductions": 48263, + "redux": 43014, + "redvelvet": 41845, + "redwings": 31058, + "redwood": 31748, + "ree": 9282, + "ree": 5813, + "reebok": 26734, + "reece": 30457, + "reed": 26209, + "reed": 10435, + "reedus": 32865, + "reef": 46557, + "reef": 15624, + "reefs": 34459, + "reel": 34467, + "reel": 17166, + "reels": 48127, + "reem": 48891, + "reen": 21638, + "reen": 23679, + "rees": 18314, + "reese": 20929, + "reeves": 23060, + "ref": 4067, + "ref": 9591, + "refe": 5624, + "refer": 18425, + "refer": 22325, + "referee": 20398, + "referees": 45583, + "referen": 13535, + "reference": 10214, + "references": 24009, + "referendum": 16732, + "referr": 47784, + "referral": 30219, + "referred": 22969, + "referring": 29797, + "refers": 30069, + "refill": 37859, + "refin": 13455, + "refined": 26098, + "refinery": 31393, + "refining": 48406, + "reflec": 4608, + "reflect": 13373, + "reflected": 28732, + "reflecting": 19700, + "reflection": 11884, + "reflections": 16647, + "reflective": 27008, + "reflects": 15821, + "reflex": 45756, + "reflex": 36050, + "reform": 45678, + "reform": 8875, + "reformation": 45119, + "reformed": 40880, + "reforms": 19274, + "refr": 34850, + "refre": 11995, + "refresh": 17836, + "refresh": 23288, + "refreshed": 35925, + "refresher": 41481, + "refreshing": 14159, + "refreshments": 31127, + "refriger": 21076, + "refrigerator": 36662, + "refs": 35595, + "refu": 3545, + "refuge": 5638, + "refuge": 17432, + "refugee": 11556, + "refugees": 42687, + "refugees": 8316, + "refund": 28899, + "refur": 15519, + "refurbi": 18259, + "refurbished": 26190, + "refurbishment": 35803, + "refusal": 46547, + "refuse": 16412, + "refused": 17190, + "refuses": 20085, + "refusing": 26704, + "reg": 5472, + "reg": 12353, + "regain": 37510, + "regal": 31512, + "regal": 25028, + "regan": 34062, + "regar": 5881, + "regard": 21801, + "regarded": 32017, + "regarding": 8493, + "regardless": 17220, + "regards": 23079, + "regatta": 26316, + "regen": 46545, + "regency": 29341, + "regeneration": 29257, + "regent": 30455, + "regents": 46710, + "regg": 12757, + "reggae": 37821, + "reggae": 15214, + "reggie": 21872, + "regi": 1608, + "regime": 11378, + "regiment": 18603, + "regin": 23287, + "regina": 16841, + "region": 16542, + "region": 4341, + "regional": 5552, + "regionals": 26043, + "regions": 14530, + "regis": 28094, + "register": 3967, + "registered": 10254, + "registering": 33510, + "registr": 29193, + "registration": 7302, + "registrations": 38423, + "registry": 30020, + "rego": 47351, + "regram": 30329, + "regrann": 48802, + "regre": 8627, + "regression": 43733, + "regret": 14374, + "regrets": 23231, + "regu": 3411, + "regui": 46722, + "regul": 11847, + "regular": 14882, + "regular": 6307, + "regularly": 17263, + "regulat": 14575, + "regulate": 33494, + "regulated": 31384, + "regulating": 48156, + "regulation": 14267, + "regulations": 16654, + "regulator": 30364, + "regulators": 35837, + "regulatory": 17717, + "reh": 21492, + "reha": 10193, + "rehab": 16973, + "rehabil": 17930, + "rehabilitation": 21042, + "rehear": 7273, + "rehearsal": 11482, + "rehearsals": 17977, + "rehearsing": 23125, + "rehman": 39206, + "rei": 15343, + "rei": 26033, + "reic": 41230, + "reich": 48589, + "reich": 28929, + "reid": 45125, + "reid": 11744, + "reig": 13092, + "reign": 41419, + "reign": 14827, + "reigning": 28409, + "reigns": 21217, + "reiki": 46960, + "reilly": 28120, + "reim": 35421, + "reimagined": 46799, + "reimbur": 39857, + "rein": 9240, + "rein": 45009, + "reina": 43847, + "reinde": 23810, + "reindeer": 25072, + "reinfor": 48161, + "reinforced": 41909, + "reinst": 33969, + "reinvent": 38171, + "reissue": 34042, + "reiter": 35394, + "rejec": 9958, + "reject": 22435, + "rejected": 17505, + "rejection": 32264, + "rejects": 23155, + "rejo": 20150, + "rejoice": 24712, + "rejuven": 26332, + "rek": 47542, + "rek": 19201, + "rel": 1825, + "rel": 5233, + "rela": 4362, + "reland": 15220, + "relat": 27192, + "relatable": 31010, + "relate": 17520, + "related": 5880, + "relates": 36064, + "relating": 27373, + "relation": 4561, + "relation": 16207, + "relations": 10100, + "relationship": 47239, + "relationship": 5837, + "relationships": 10610, + "relative": 17265, + "relatively": 18351, + "relatives": 21981, + "relax": 6777, + "relax": 9035, + "relaxation": 22194, + "relaxed": 18999, + "relaxing": 10256, + "relay": 12403, + "relays": 28404, + "rele": 1602, + "release": 29100, + "release": 2706, + "released": 3410, + "releases": 7393, + "releasethe": 44008, + "releasing": 10321, + "releg": 23378, + "relegated": 45884, + "relegation": 35040, + "relent": 22213, + "relentless": 27207, + "relessly": 33927, + "relev": 9349, + "relevance": 31400, + "relevant": 10568, + "reli": 2674, + "reliability": 27220, + "reliable": 13714, + "reliance": 27727, + "relic": 27802, + "relics": 43208, + "relief": 7518, + "relies": 41579, + "relieve": 28623, + "relieved": 36597, + "religi": 4940, + "religion": 8803, + "religions": 31189, + "religious": 8289, + "relish": 35550, + "relive": 23939, + "reliving": 47558, + "rell": 28802, + "rell": 7127, + "rella": 9952, + "relle": 31390, + "reloaded": 38908, + "relocated": 46791, + "relocation": 39198, + "rels": 23320, + "relu": 32058, + "reluct": 32549, + "reluctant": 45552, + "rely": 4158, + "relying": 42168, + "rem": 15098, + "rem": 21637, + "rema": 4569, + "remain": 29144, + "remain": 6415, + "remainder": 41672, + "remained": 23714, + "remaining": 11392, + "remains": 6807, + "remake": 16234, + "remark": 11136, + "remarkable": 12404, + "remarkably": 39087, + "remarks": 15001, + "remastered": 24932, + "rematch": 26473, + "rembrandt": 45972, + "reme": 20071, + "remedi": 18442, + "remedies": 25581, + "remedy": 25794, + "remem": 7966, + "rememb": 7062, + "remember": 22045, + "remember": 2195, + "remembered": 11763, + "remembering": 8135, + "remembers": 12551, + "remembrance": 40321, + "remembrance": 15860, + "remembranceday": 48333, + "rement": 7173, + "rements": 12667, + "remi": 41693, + "remin": 3216, + "remind": 9868, + "reminded": 12309, + "reminder": 5565, + "reminders": 34121, + "reminding": 19976, + "reminds": 8303, + "remington": 43527, + "reminis": 17723, + "reminiscent": 41704, + "reminiscing": 32552, + "remix": 8519, + "remixes": 31011, + "remn": 29127, + "remnants": 39032, + "remo": 4064, + "remo": 33259, + "remodel": 34159, + "remodel": 37495, + "remodeling": 41432, + "remote": 47163, + "remote": 9687, + "remotely": 32375, + "removable": 44095, + "removal": 13679, + "remove": 9709, + "removed": 10289, + "remover": 44267, + "removes": 29018, + "removing": 18504, + "remy": 30434, + "ren": 737, + "ren": 2596, + "rena": 12591, + "renais": 15409, + "renaissance": 16007, + "renal": 36096, + "renamed": 31535, + "renault": 17600, + "rence": 19245, + "rence": 1553, + "rences": 8545, + "rend": 33932, + "rend": 22851, + "render": 39752, + "render": 13024, + "rendered": 23652, + "rendering": 21339, + "renders": 39419, + "rendez": 43293, + "rendezvous": 45644, + "rendition": 28891, + "rendon": 46272, + "rendous": 49403, + "rends": 38842, + "rene": 15438, + "rene": 12597, + "renee": 23480, + "reneg": 29909, + "renegade": 41229, + "renergy": 37151, + "renew": 6645, + "renew": 22015, + "renewable": 31269, + "renewable": 15941, + "renewableenergy": 33357, + "renewables": 21619, + "renewal": 21270, + "renewed": 20524, + "renfre": 45043, + "reng": 36795, + "reno": 11520, + "reno": 12831, + "renov": 9984, + "renovated": 23839, + "renovation": 17121, + "renovations": 31311, + "renowned": 14727, + "rens": 18183, + "renshaw": 44445, + "rent": 17377, + "rent": 1609, + "rental": 12193, + "rentals": 24105, + "rented": 35932, + "rential": 31692, + "renting": 37662, + "rently": 2615, + "rents": 31109, + "reo": 15963, + "reo": 26854, + "reon": 15761, + "reopen": 26883, + "reopened": 32868, + "reopening": 36663, + "reopens": 40644, + "rep": 4229, + "rep": 6487, + "repair": 8419, + "repaired": 32953, + "repairing": 38534, + "repairs": 16297, + "repar": 34065, + "repe": 5785, + "repeal": 42622, + "repeal": 23938, + "repeat": 10192, + "repeated": 27904, + "repeatedly": 26630, + "repeating": 33834, + "repeats": 39158, + "repell": 46235, + "repent": 47261, + "reper": 29085, + "repet": 38533, + "repl": 13047, + "replac": 6069, + "replace": 9466, + "replaceable": 47762, + "replaced": 13200, + "replacement": 10835, + "replaces": 27781, + "replacing": 18647, + "replay": 16875, + "repleni": 44839, + "replic": 21651, + "replica": 18125, + "replied": 24238, + "replies": 18808, + "reply": 8965, + "replying": 47599, + "repor": 2628, + "report": 2417, + "reported": 7598, + "reportedly": 10953, + "reporter": 11019, + "reporters": 18454, + "reporting": 9218, + "reports": 4908, + "reposit": 41276, + "repository": 46977, + "repost": 33147, + "repost": 7217, + "repostapp": 38388, + "reposting": 20223, + "reppin": 19163, + "repping": 22574, + "repre": 3397, + "represent": 8293, + "represent": 8406, + "representation": 13520, + "representative": 13175, + "representatives": 15591, + "represented": 12299, + "representing": 7561, + "represents": 14433, + "repri": 31854, + "reproduction": 35714, + "reproductive": 25522, + "reps": 14265, + "reptile": 36938, + "reptiles": 38679, + "republic": 6376, + "republic": 7185, + "republican": 9842, + "republicans": 12384, + "repur": 41852, + "req": 42411, + "requ": 10664, + "reque": 9539, + "request": 7813, + "requested": 16199, + "requesting": 33245, + "requests": 17087, + "requi": 4863, + "requiem": 40316, + "require": 14437, + "required": 8500, + "requirement": 27146, + "requirements": 12860, + "requires": 13396, + "requiring": 33425, + "requis": 42602, + "rer": 41295, + "rer": 3407, + "rera": 14301, + "rero": 21860, + "rers": 18869, + "res": 4466, + "res": 934, + "resc": 3956, + "rescheduled": 43553, + "rescu": 8618, + "rescue": 28567, + "rescue": 5718, + "rescued": 11919, + "rescues": 32439, + "rescuing": 43770, + "rese": 13000, + "resear": 6090, + "research": 25694, + "research": 2379, + "researched": 42733, + "researcher": 18334, + "researchers": 9522, + "researching": 24544, + "reseller": 35391, + "resemb": 16916, + "resemblance": 26856, + "resemble": 37230, + "resembles": 35417, + "reser": 16420, + "reserv": 11906, + "reservation": 20289, + "reservations": 19307, + "reserve": 6911, + "reserved": 19796, + "reserves": 19705, + "reservoir": 20574, + "reset": 26250, + "resh": 47432, + "reshi": 39435, + "resi": 2152, + "residen": 22311, + "residence": 11672, + "residences": 38855, + "residency": 18545, + "resident": 9016, + "residente": 44637, + "residentevil": 48393, + "residential": 11002, + "residents": 6008, + "resign": 23584, + "resignation": 24779, + "resigned": 31014, + "resigns": 29738, + "resil": 10932, + "resili": 39212, + "resilience": 15271, + "resilient": 24694, + "resin": 24156, + "resist": 37345, + "resist": 9587, + "resistance": 7392, + "resistant": 17542, + "resisting": 43679, + "resolution": 9977, + "resolutions": 26816, + "resolve": 20787, + "resolved": 28807, + "reson": 18092, + "resonance": 42310, + "resort": 6594, + "resorts": 18839, + "resource": 43729, + "resource": 9760, + "resources": 6723, + "respec": 7466, + "respect": 31411, + "respect": 4916, + "respected": 19126, + "respectful": 24379, + "respecting": 36172, + "respective": 25817, + "respectively": 28794, + "respects": 23553, + "respir": 20771, + "respiratory": 24483, + "respon": 2421, + "respond": 12355, + "responded": 21121, + "respondents": 49253, + "responders": 25155, + "responding": 18037, + "responds": 17436, + "response": 5399, + "responses": 19006, + "responsi": 5490, + "responsibilities": 30375, + "responsibility": 11272, + "responsible": 8936, + "responsibly": 33675, + "responsive": 21544, + "ress": 34651, + "ress": 13629, + "resso": 15133, + "rest": 10974, + "rest": 2539, + "restart": 37378, + "restaur": 3775, + "restaurant": 41930, + "restaurant": 4489, + "restaurants": 11714, + "rested": 46020, + "resting": 18044, + "restless": 36724, + "restling": 30076, + "resto": 11118, + "resto": 41666, + "restock": 34060, + "restocked": 36966, + "restor": 8984, + "restoration": 11989, + "restorative": 46509, + "restore": 14008, + "restored": 14238, + "restoring": 24406, + "restra": 25424, + "restric": 11036, + "restricted": 27197, + "restriction": 44282, + "restrictions": 19884, + "restroom": 43423, + "restructuring": 43260, + "rests": 33775, + "resu": 10095, + "resul": 2655, + "result": 5659, + "resulted": 26449, + "resulting": 24581, + "results": 3790, + "resume": 15077, + "resumes": 30268, + "resur": 14865, + "resurg": 45962, + "resurgence": 47692, + "resurrec": 18487, + "resurrection": 25811, + "resusc": 47523, + "ret": 20500, + "ret": 10048, + "reta": 20153, + "retail": 14910, + "retail": 6455, + "retailer": 22549, + "retailers": 19418, + "retain": 24430, + "retained": 42737, + "retaining": 35571, + "retains": 42583, + "retali": 33101, + "retar": 29964, + "retarded": 44111, + "retention": 26247, + "rethink": 29078, + "rethinking": 42951, + "reti": 4721, + "retin": 31270, + "retina": 36919, + "retire": 18846, + "retired": 11477, + "retirement": 9205, + "retires": 29060, + "retiring": 21200, + "retrac": 32735, + "retreat": 11210, + "retri": 16918, + "retriever": 28394, + "retro": 6535, + "retro": 7755, + "retrogamer": 47220, + "retrogaming": 11316, + "retrospective": 27105, + "rett": 41082, + "rett": 8425, + "rette": 33066, + "return": 43042, + "return": 3458, + "returned": 10476, + "returning": 9290, + "returns": 5020, + "retwee": 48190, + "retweet": 3195, + "retweeted": 12705, + "retweeting": 32345, + "retweets": 10160, + "rety": 41550, + "reu": 20255, + "reu": 40371, + "reuben": 40450, + "reunion": 10247, + "reunite": 26179, + "reunited": 13516, + "reusable": 30395, + "reuse": 26535, + "reut": 15210, + "reuters": 15569, + "rev": 8424, + "rev": 11789, + "revamp": 29819, + "revamped": 36420, + "revan": 45277, + "reve": 3115, + "reveal": 8052, + "revealed": 7171, + "revealing": 21321, + "reveals": 6621, + "revel": 14133, + "revelation": 24053, + "revelations": 36163, + "reven": 10171, + "revenge": 12717, + "revenue": 10637, + "revenues": 33348, + "rever": 14829, + "rever": 41913, + "revere": 44187, + "reverend": 34407, + "revers": 20726, + "reversal": 33367, + "reverse": 12812, + "reversed": 42485, + "reversi": 31601, + "reversible": 34212, + "revi": 8317, + "review": 2268, + "reviewed": 16678, + "reviewer": 36409, + "reviewers": 48195, + "reviewing": 20458, + "reviews": 7227, + "revise": 46801, + "revised": 22806, + "revising": 46882, + "revision": 20335, + "revisit": 26568, + "revisited": 34302, + "revisiting": 33144, + "revit": 26367, + "revitalization": 46923, + "revival": 14142, + "revive": 26450, + "revived": 42912, + "revo": 28660, + "revol": 13447, + "revolt": 31697, + "revolu": 4900, + "revolution": 17699, + "revolution": 6644, + "revolutionary": 14734, + "revolver": 38747, + "revolving": 47230, + "revs": 49286, + "revue": 43428, + "rew": 37564, + "rewar": 15857, + "reward": 11223, + "rewarded": 27163, + "rewarding": 23351, + "rewards": 15235, + "rewatch": 35610, + "rewatching": 41287, + "rewind": 26867, + "rewrite": 45218, + "rex": 13002, + "rex": 10904, + "rexperience": 33924, + "rey": 9681, + "rey": 4517, + "reyes": 18255, + "reykja": 47571, + "reyn": 11998, + "reynolds": 14309, + "reys": 48284, + "rez": 27597, + "rez": 15192, + "reza": 35888, + "rf": 35529, + "rf": 16368, + "rfc": 19003, + "rfid": 40204, + "rg": 33055, + "rg": 14897, + "rgb": 36128, + "rgv": 33685, + "rh": 8745, + "rh": 22404, + "rha": 19473, + "rhapso": 32532, + "rhapsody": 35774, + "rhe": 9186, + "rhea": 28612, + "rhetor": 24359, + "rhetoric": 29985, + "rhett": 42984, + "rheu": 42953, + "rhi": 21212, + "rhin": 12269, + "rhine": 22863, + "rhine": 44833, + "rhinestone": 30450, + "rhino": 41744, + "rhino": 20056, + "rhinos": 30671, + "rho": 7637, + "rhode": 39302, + "rhode": 27907, + "rhodes": 17785, + "rhon": 25882, + "rhonda": 46100, + "rhp": 27199, + "rhs": 24551, + "rhu": 23897, + "rhubarb": 30213, + "rhy": 7740, + "rhyme": 37356, + "rhymes": 33143, + "rhys": 28647, + "rhyth": 27069, + "rhythm": 16172, + "rhythmic": 46386, + "rhythms": 40872, + "ri": 553, + "ri": 2574, + "ria": 3650, + "rial": 15200, + "rian": 7788, + "rib": 44634, + "rib": 18298, + "riba": 44992, + "ribb": 10081, + "ribbon": 12114, + "ribbons": 35271, + "ribe": 46115, + "ribs": 17519, + "ric": 920, + "ric": 4798, + "rica": 14230, + "rical": 18109, + "rican": 30958, + "ricardo": 23140, + "ricci": 35783, + "ricciardo": 49282, + "rice": 36362, + "rice": 4741, + "rich": 5223, + "rich": 4021, + "richar": 9350, + "richard": 9080, + "richard": 4470, + "richards": 11372, + "richardson": 15984, + "riche": 23286, + "richer": 34138, + "riches": 37093, + "richest": 25572, + "richi": 38934, + "richie": 19797, + "richland": 43079, + "richmond": 34143, + "richmond": 11292, + "richter": 37591, + "rick": 6237, + "rick": 3064, + "ricket": 46161, + "ricket": 23671, + "ricks": 23111, + "ricky": 19188, + "ricky": 12814, + "rico": 37962, + "rico": 11362, + "ricotta": 38473, + "rics": 7353, + "ricul": 6980, + "rid": 18103, + "rid": 9874, + "ridd": 21990, + "ridden": 32025, + "riddle": 31839, + "ride": 15816, + "ride": 2994, + "rider": 31056, + "rider": 9707, + "riders": 10826, + "rides": 11308, + "ridg": 42646, + "ridge": 16580, + "ridge": 6352, + "ridic": 9624, + "ridiculous": 12659, + "ridiculously": 25661, + "ridin": 47869, + "riding": 6765, + "ridley": 27883, + "rie": 14824, + "rie": 5322, + "ried": 7552, + "riel": 26696, + "rien": 35237, + "rier": 40714, + "rier": 13336, + "ries": 28179, + "ries": 3059, + "riesling": 36372, + "rif": 7044, + "riff": 30359, + "rifle": 15354, + "rifles": 25678, + "rift": 26681, + "rig": 18462, + "rig": 13871, + "riga": 36626, + "rigged": 35897, + "rigging": 38160, + "riggs": 40328, + "righ": 15391, + "right": 13341, + "right": 1155, + "righte": 20762, + "righteous": 28169, + "righteousness": 42481, + "rightful": 42601, + "rightly": 42669, + "rights": 3336, + "rigid": 43138, + "rigor": 36788, + "rigorous": 41654, + "rigs": 42893, + "rihanna": 13744, + "rij": 41097, + "rik": 31136, + "rik": 27832, + "rika": 28580, + "ril": 12270, + "ril": 2388, + "riley": 35056, + "riley": 12260, + "rill": 23705, + "rilla": 43956, + "rilla": 18685, + "rim": 28147, + "rim": 12199, + "rime": 27064, + "rimin": 11527, + "rimo": 47817, + "rims": 34327, + "rin": 5859, + "rin": 11739, + "rina": 12869, + "rine": 24952, + "ring": 8318, + "ring": 2540, + "ringed": 44712, + "ringer": 35761, + "ringing": 26035, + "ringo": 38845, + "rings": 5751, + "rington": 12455, + "rink": 21497, + "rinka": 47316, + "rino": 47188, + "rinse": 48320, + "rio": 15681, + "rio": 5782, + "rion": 31623, + "rion": 34046, + "rios": 32814, + "riot": 32636, + "riot": 14218, + "riots": 24844, + "rious": 6340, + "rip": 10353, + "rip": 4243, + "ripe": 22832, + "ripley": 41589, + "ripp": 25276, + "ripped": 17815, + "ripper": 35347, + "ripping": 29126, + "ripple": 24825, + "rips": 30182, + "rir": 36792, + "ris": 6108, + "ris": 1999, + "rise": 13641, + "rise": 3151, + "risen": 23653, + "risers": 44983, + "rises": 13362, + "riseup": 35760, + "rish": 18378, + "rish": 18927, + "rishi": 48434, + "rising": 30452, + "rising": 5448, + "risis": 37998, + "risk": 27967, + "risk": 4213, + "risking": 48155, + "risks": 12474, + "risky": 27630, + "risotto": 31471, + "rist": 40610, + "rit": 5156, + "rit": 17333, + "rita": 16178, + "ritchie": 30997, + "rite": 39318, + "rite": 18429, + "rites": 36160, + "rith": 48169, + "rith": 48850, + "riti": 32904, + "rito": 19379, + "ritos": 33507, + "ritt": 26092, + "ritter": 34854, + "ritu": 13391, + "ritual": 19712, + "rituals": 31145, + "ritz": 39151, + "ritz": 25627, + "rium": 33884, + "riv": 25113, + "rival": 13412, + "rival": 15629, + "rivalry": 19511, + "rivals": 15135, + "rive": 27588, + "rive": 34917, + "river": 5239, + "river": 2473, + "rivera": 18275, + "riverdale": 28304, + "riverfront": 44439, + "rivers": 10723, + "riverside": 15809, + "riveting": 44024, + "riviera": 25851, + "rix": 43407, + "rix": 9483, + "riya": 36908, + "riyad": 31564, + "riyadh": 33577, + "riz": 18426, + "riz": 35411, + "rizal": 41555, + "rizio": 40191, + "rizz": 34826, + "rizzo": 49076, + "rj": 26016, + "rj": 20949, + "rk": 38725, + "rk": 21422, + "rl": 18041, + "rl": 14590, + "rlly": 43222, + "rly": 25954, + "rm": 20202, + "rm": 8431, + "rmb": 49097, + "rms": 40529, + "rn": 13206, + "rn": 7666, + "rna": 24566, + "rnb": 31556, + "rnc": 35309, + "rnli": 29748, + "ro": 532, + "ro": 2795, + "roa": 8313, + "roach": 31073, + "road": 4370, + "road": 1759, + "roadhouse": 47891, + "roadmap": 30111, + "roads": 6189, + "roadsafety": 39992, + "roadshow": 21168, + "roadside": 26928, + "roadster": 28920, + "roadto": 24681, + "roadtrip": 15094, + "roadway": 42744, + "roam": 34045, + "roaming": 29240, + "roano": 34184, + "roanoke": 36587, + "roar": 34193, + "roar": 18483, + "roaring": 26428, + "roast": 11404, + "roasted": 10479, + "roasting": 32228, + "rob": 2668, + "rob": 6442, + "robb": 14059, + "robb": 39673, + "robbed": 24163, + "robber": 35545, + "robbers": 40852, + "robbery": 16393, + "robbi": 44898, + "robbie": 37200, + "robbie": 15970, + "robbing": 47569, + "robbins": 23461, + "robby": 44128, + "robe": 23116, + "rober": 4532, + "robert": 8811, + "robert": 3929, + "roberta": 43373, + "roberto": 42645, + "roberto": 16227, + "roberts": 10366, + "robertson": 17643, + "robes": 29304, + "robi": 16743, + "robin": 6681, + "robin": 7988, + "robins": 35502, + "robinson": 8523, + "robles": 47646, + "roblo": 27481, + "roblox": 37798, + "robo": 4672, + "robo": 36057, + "robot": 46089, + "robot": 8797, + "robotic": 23975, + "robotics": 13546, + "robots": 13473, + "robson": 31113, + "robust": 22780, + "robyn": 34533, + "roc": 3268, + "roc": 13776, + "rocco": 30009, + "roch": 23788, + "rochdale": 41880, + "roche": 31776, + "rochelle": 40161, + "rochester": 18057, + "rock": 2640, + "rock": 2172, + "rockab": 39353, + "rockabilly": 45019, + "rocke": 19914, + "rocked": 16116, + "rockefeller": 35476, + "rocker": 29008, + "rockers": 32338, + "rocket": 25435, + "rocket": 8383, + "rockets": 13292, + "rockford": 41039, + "rockies": 20621, + "rockin": 12073, + "rocking": 7081, + "rockn": 24442, + "rocknroll": 27840, + "rocks": 6135, + "rockstar": 23603, + "rockstar": 18000, + "rockstargames": 27516, + "rockstars": 46639, + "rockthe": 49363, + "rockwell": 34747, + "rocky": 33481, + "rocky": 9648, + "rod": 9712, + "rod": 8291, + "roddy": 42332, + "rode": 18449, + "rodeo": 18250, + "rodgers": 17612, + "rodi": 49100, + "rodney": 21753, + "rodri": 11053, + "rodrigo": 33944, + "rodriguez": 14057, + "rods": 28618, + "roe": 27671, + "roe": 9996, + "rof": 33029, + "rofl": 48228, + "roft": 45212, + "rog": 34269, + "rog": 34017, + "rogen": 23380, + "roger": 13929, + "roger": 7735, + "rogerfederer": 40182, + "rogers": 10661, + "rogue": 32575, + "rogue": 15162, + "roh": 14933, + "roh": 29840, + "rohan": 39848, + "rohing": 23600, + "rohingya": 26146, + "rohit": 44649, + "rohit": 24299, + "roi": 21877, + "rok": 36807, + "rol": 3393, + "rol": 7818, + "roland": 33713, + "roland": 19569, + "role": 18485, + "role": 3414, + "roles": 11871, + "rolex": 21093, + "rolf": 48606, + "roll": 4711, + "roll": 3341, + "rolled": 11982, + "roller": 21034, + "roller": 12342, + "rollercoaster": 38248, + "rollers": 36941, + "rollin": 27545, + "rolling": 24250, + "rolling": 6347, + "rollingstones": 41309, + "rollins": 27724, + "rollout": 47710, + "rollover": 39214, + "rolls": 8614, + "rolltide": 28101, + "rom": 11377, + "rom": 19205, + "roma": 44134, + "roma": 11631, + "romain": 48897, + "roman": 4416, + "roman": 7370, + "romance": 7215, + "romania": 15884, + "romanian": 30866, + "romano": 38409, + "romans": 23066, + "romantic": 41457, + "romantic": 8821, + "rome": 9406, + "rome": 5243, + "romeo": 14429, + "romero": 23694, + "romney": 19287, + "romo": 32248, + "romper": 43699, + "ron": 2393, + "ron": 3372, + "rona": 42385, + "ronal": 46194, + "ronald": 15683, + "ronaldo": 13463, + "ronan": 34971, + "rond": 31935, + "ronda": 37436, + "rondo": 43756, + "rone": 48082, + "rone": 32763, + "roni": 47234, + "ronnie": 45257, + "ronnie": 16421, + "rons": 19536, + "ront": 48881, + "roo": 1249, + "roo": 31227, + "rood": 38007, + "roof": 9120, + "roof": 6449, + "roofing": 24415, + "roofs": 34635, + "rooftop": 16319, + "rook": 35918, + "rookie": 9771, + "rookies": 31917, + "room": 8845, + "room": 1530, + "roomie": 36851, + "roommate": 19825, + "roommates": 37323, + "rooms": 6328, + "rooney": 17712, + "roos": 32938, + "roosevel": 17644, + "roosevelt": 18488, + "rooster": 46263, + "rooster": 30926, + "roosters": 43693, + "root": 25930, + "root": 9728, + "rooted": 30428, + "rooting": 25523, + "roots": 8084, + "rop": 43401, + "rope": 9953, + "ropes": 30506, + "ror": 8668, + "ror": 2843, + "rors": 12072, + "rory": 42804, + "rory": 17813, + "ros": 5288, + "ros": 6930, + "rosa": 14393, + "rosal": 30397, + "rosario": 33640, + "rosary": 33098, + "rosberg": 46037, + "rose": 6146, + "rose": 3568, + "roseanne": 47528, + "rosel": 33616, + "rosemary": 19472, + "rosen": 13214, + "rosen": 36424, + "rosenberg": 43558, + "rosenthal": 46990, + "roses": 9061, + "rosetta": 43800, + "rosewood": 38686, + "rosie": 43049, + "rosie": 16888, + "ross": 8801, + "ross": 2158, + "rosse": 11602, + "rossi": 24817, + "rosso": 33023, + "roster": 12487, + "roswell": 45116, + "rosy": 46705, + "rosé": 28006, + "rot": 10055, + "rot": 9643, + "rotar": 45959, + "rotary": 14654, + "rotating": 32265, + "rotation": 18089, + "rotc": 32252, + "roth": 17741, + "roth": 19139, + "rother": 23174, + "rotherham": 37687, + "rothschild": 45089, + "roti": 46940, + "roto": 34698, + "rotor": 42991, + "rots": 16642, + "rott": 34806, + "rotten": 24324, + "rotter": 22614, + "rotterdam": 23422, + "rotun": 42970, + "rou": 2964, + "rou": 34783, + "roud": 28375, + "rouge": 16209, + "rough": 11699, + "rough": 8511, + "roughly": 21910, + "roughs": 37598, + "rouhani": 39912, + "roulette": 39930, + "roun": 5602, + "round": 9403, + "round": 2522, + "roundabout": 29953, + "rounded": 26973, + "rounder": 37024, + "rounding": 40208, + "rounds": 11242, + "roundtable": 19386, + "roundup": 17503, + "roup": 29220, + "rourke": 38753, + "rous": 33645, + "rous": 34531, + "rousey": 46267, + "rout": 7502, + "rout": 41778, + "route": 5261, + "router": 29962, + "routes": 14923, + "routine": 12319, + "routines": 44074, + "routing": 44086, + "roux": 43416, + "rov": 23971, + "rove": 30130, + "rover": 12776, + "rovers": 16373, + "row": 5275, + "row": 1044, + "rowan": 26240, + "rowdy": 32141, + "rowe": 28323, + "rowed": 22615, + "rower": 43345, + "rowers": 41806, + "rowing": 12807, + "rowland": 33037, + "rowley": 48793, + "rowling": 29371, + "rown": 22287, + "rown": 25060, + "rows": 9409, + "rox": 14111, + "rox": 41033, + "roxy": 28093, + "roy": 2128, + "roy": 6354, + "royal": 6691, + "royal": 3853, + "royale": 20630, + "royalnavy": 41545, + "royals": 13335, + "royalties": 48660, + "royalty": 18296, + "royalwedding": 27461, + "royce": 18444, + "royd": 41476, + "royo": 39357, + "roz": 28989, + "roz": 37250, + "rp": 17305, + "rp": 8174, + "rpa": 41872, + "rpg": 12445, + "rpm": 23715, + "rps": 49215, + "rr": 5311, + "rr": 9126, + "rrp": 36967, + "rrr": 18267, + "rrrr": 25561, + "rrrr": 34444, + "rs": 6978, + "rs": 1724, + "rsa": 29437, + "rsc": 48524, + "rsd": 34426, + "rsi": 39046, + "rsl": 44752, + "rsp": 16381, + "rspb": 38508, + "rspb": 36727, + "rspca": 45643, + "rss": 46466, + "rss": 22350, + "rstats": 38700, + "rsvp": 9774, + "rt": 8959, + "rt": 8991, + "rtc": 31648, + "rte": 33822, + "rte": 23322, + "rtg": 22028, + "rti": 47549, + "rtr": 43999, + "rts": 8496, + "rtw": 34673, + "ru": 681, + "ru": 13735, + "rub": 15862, + "rub": 22586, + "rubb": 19597, + "rubbed": 45239, + "rubber": 31131, + "rubber": 11331, + "rubbing": 41262, + "rubbish": 21108, + "rubble": 42230, + "ruben": 44058, + "ruben": 29722, + "rubi": 27856, + "rubin": 34128, + "rubio": 24244, + "rubs": 43422, + "ruby": 24552, + "ruby": 11493, + "ruck": 27449, + "rucker": 45402, + "rud": 35256, + "rudd": 31836, + "rude": 16548, + "rudi": 48360, + "rudol": 40927, + "rudolf": 46835, + "rudolph": 30119, + "rudy": 38226, + "rudy": 22131, + "rue": 38024, + "rue": 19276, + "rufc": 45084, + "ruff": 28177, + "ruff": 30304, + "rufus": 39322, + "rug": 4217, + "rug": 19220, + "rugby": 15091, + "rugby": 4964, + "rugbyleague": 44419, + "ruger": 48655, + "rugged": 25225, + "rugs": 29946, + "rui": 46974, + "ruin": 16256, + "ruined": 17231, + "ruining": 29952, + "ruins": 16094, + "ruiz": 27873, + "ruk": 46628, + "rukh": 43075, + "rukh": 27631, + "rule": 31643, + "rule": 6175, + "ruled": 16324, + "ruler": 26286, + "rulers": 45328, + "rules": 5272, + "ruling": 14690, + "rum": 9223, + "rum": 11233, + "rumb": 42432, + "rumble": 18900, + "rumi": 31428, + "rumor": 22254, + "rumored": 36694, + "rumors": 16160, + "rumour": 34296, + "rumours": 20716, + "rump": 29366, + "run": 1639, + "run": 1934, + "runaway": 28851, + "runchat": 25838, + "rundown": 41100, + "rune": 33882, + "rune": 49244, + "runner": 37370, + "runner": 7913, + "runners": 10571, + "runnin": 43130, + "running": 24451, + "running": 2761, + "runoff": 38564, + "runs": 5586, + "runway": 13927, + "rup": 7996, + "rup": 14980, + "rupaul": 44211, + "rupee": 43916, + "rupees": 44110, + "rupert": 25625, + "rupt": 23055, + "ruption": 35403, + "rural": 28801, + "rural": 8737, + "rus": 35811, + "rus": 5998, + "rush": 12148, + "rush": 6973, + "rushed": 28104, + "rusher": 48745, + "rushes": 47217, + "rushing": 20284, + "russ": 6285, + "russ": 20764, + "russell": 26122, + "russell": 8150, + "russi": 2600, + "russia": 4018, + "russian": 30731, + "russian": 4868, + "russians": 25413, + "russo": 30679, + "rust": 28682, + "rust": 14212, + "rustic": 19822, + "rusty": 43966, + "rusty": 22646, + "rut": 14973, + "rut": 39102, + "rutger": 49029, + "rutgers": 28934, + "ruth": 15798, + "ruth": 12029, + "ruther": 26676, + "rutherford": 31070, + "ruthless": 36063, + "rutland": 46024, + "ruto": 43702, + "ruz": 23275, + "rv": 17135, + "rv": 17951, + "rva": 24278, + "rw": 9085, + "rw": 22926, + "rwa": 47452, + "rwand": 31758, + "rwanda": 15427, + "rwby": 39698, + "rwc": 32321, + "rx": 41188, + "rx": 15945, + "ry": 1511, + "ry": 913, + "ryan": 8682, + "ryan": 4053, + "ryanair": 43526, + "ryder": 43564, + "ryder": 21805, + "rye": 24015, + "rye": 17409, + "rying": 7838, + "ryn": 37728, + "ryo": 24460, + "rys": 21654, + "ryu": 46656, + "ryu": 34604, + "ré": 29106, + "s": 82, + "s": 338, + "sa": 774, + "sa": 1344, + "saa": 13429, + "saab": 27158, + "saad": 36530, + "saas": 25761, + "saat": 33151, + "sab": 3233, + "sab": 23213, + "saba": 38344, + "sabah": 32854, + "saban": 41620, + "sabar": 47102, + "sabbath": 26008, + "sabc": 30010, + "sabcnews": 41093, + "saber": 46822, + "saber": 25624, + "sabha": 23431, + "sabi": 47073, + "sabine": 44062, + "sable": 19224, + "sabot": 30700, + "sabotage": 40496, + "sabre": 35110, + "sabres": 29620, + "sabrin": 37029, + "sabrina": 24994, + "sac": 3632, + "sac": 12905, + "sach": 30168, + "sacha": 49010, + "sachin": 47527, + "sachin": 30297, + "sachs": 31451, + "sack": 28964, + "sack": 14979, + "sacked": 27519, + "sacks": 26441, + "sacram": 13334, + "sacramento": 16065, + "sacred": 40612, + "sacred": 12477, + "sacri": 15283, + "sacrif": 12117, + "sacrific": 16919, + "sacrifice": 12556, + "sacrificed": 31116, + "sacrifices": 28858, + "sacrificing": 48146, + "sad": 2810, + "sad": 3719, + "saddened": 27720, + "saddest": 34925, + "saddle": 30469, + "saddle": 20283, + "sade": 27429, + "sadh": 40955, + "sadi": 22207, + "sadie": 30333, + "sadiq": 44107, + "sadler": 45600, + "sadly": 11603, + "sadness": 20399, + "sae": 38633, + "sae": 34883, + "saeed": 29745, + "saf": 2125, + "saf": 25760, + "safar": 23443, + "safari": 14091, + "safarilive": 34816, + "safc": 27998, + "safe": 2901, + "safe": 2996, + "safeguard": 42249, + "safeguarding": 47451, + "safely": 11513, + "safer": 40124, + "safer": 15504, + "safest": 38973, + "safety": 19050, + "safety": 3406, + "safetyfirst": 43608, + "saffron": 27529, + "sag": 6609, + "sag": 30048, + "saga": 15758, + "sagan": 37193, + "sagar": 42518, + "sage": 25800, + "sage": 7509, + "sages": 25979, + "sagin": 47097, + "sagitt": 44685, + "sagu": 44708, + "sah": 30943, + "sah": 26342, + "saha": 36062, + "sahara": 24599, + "saharan": 44255, + "sahi": 24608, + "sahib": 34150, + "sai": 16048, + "sai": 10886, + "said": 40319, + "said": 1946, + "saif": 44164, + "saig": 36328, + "saigon": 41081, + "sail": 7528, + "sail": 12156, + "sailed": 43047, + "sailing": 11003, + "sailor": 28002, + "sailor": 16076, + "sailormoon": 40673, + "sailors": 25355, + "sails": 27526, + "sain": 21226, + "sain": 40378, + "sains": 24860, + "sainsbury": 45879, + "sainsburys": 36934, + "saint": 11274, + "saint": 5599, + "saints": 8769, + "saintsfc": 31102, + "sair": 46600, + "sair": 30971, + "saire": 28087, + "saison": 33256, + "sait": 48008, + "saj": 33580, + "sak": 11511, + "sak": 35900, + "saka": 33609, + "sake": 12874, + "sakh": 43945, + "saki": 40514, + "saku": 37550, + "sakura": 24162, + "sal": 980, + "sal": 6126, + "sala": 17300, + "salaam": 46773, + "salad": 6188, + "salads": 30948, + "salah": 22516, + "salam": 19007, + "salam": 33963, + "salamat": 44696, + "salami": 46885, + "salaries": 33132, + "salary": 16312, + "salazar": 45988, + "sale": 17786, + "sale": 1690, + "saleh": 38353, + "salem": 48194, + "salem": 16884, + "sales": 13347, + "sales": 3765, + "salesforce": 22680, + "salesman": 37633, + "salford": 25629, + "sali": 15411, + "salim": 42760, + "salinas": 41990, + "saline": 46918, + "salis": 20667, + "salis": 39378, + "salisbury": 24763, + "sall": 27122, + "sall": 20883, + "salle": 23738, + "sally": 29542, + "sally": 13349, + "salman": 13754, + "salman": 16219, + "salmankhan": 15177, + "salmon": 37040, + "salmon": 9137, + "salom": 38268, + "salon": 33916, + "salon": 11105, + "saloon": 26038, + "sals": 16307, + "salsa": 16442, + "salt": 12763, + "salt": 6611, + "salted": 26313, + "saltlife": 47809, + "salts": 40559, + "saltwater": 43616, + "salty": 20678, + "salu": 31711, + "salud": 46867, + "salut": 44998, + "salute": 44908, + "salute": 9747, + "salutes": 32762, + "salv": 8299, + "salvador": 20874, + "salvage": 33131, + "salvation": 19534, + "salvatore": 38772, + "salz": 33594, + "salzburg": 43396, + "sam": 1644, + "sam": 3730, + "sama": 19272, + "samanth": 11465, + "samantha": 15466, + "samanthap": 38266, + "samanthaprabhu": 38643, + "samar": 21820, + "samaritan": 45495, + "samba": 37190, + "same": 23062, + "same": 2208, + "samheughan": 36255, + "sami": 48400, + "sami": 24322, + "sammy": 31091, + "sammy": 16758, + "samo": 30006, + "samoa": 34932, + "samp": 31225, + "sample": 9542, + "sampler": 40629, + "samples": 13387, + "sampling": 19522, + "sampson": 39983, + "sams": 44667, + "samson": 34659, + "samsun": 47875, + "samsung": 35369, + "samsung": 8115, + "samu": 7646, + "samuel": 30612, + "samuel": 12787, + "samurai": 21739, + "san": 1489, + "san": 2223, + "sana": 19434, + "sanantonio": 34714, + "sanat": 29091, + "sanatomy": 36052, + "sanc": 7398, + "sance": 15930, + "sanchez": 13971, + "sanctioned": 43032, + "sanctions": 17790, + "sanctu": 12712, + "sanctuary": 14044, + "sand": 2147, + "sand": 5094, + "sandal": 36445, + "sandal": 42185, + "sandals": 20731, + "sandalwood": 47502, + "sandeep": 46973, + "sander": 34111, + "sanders": 10429, + "sanderson": 36198, + "sandi": 44249, + "sandiego": 45997, + "sandiego": 15793, + "sandman": 45730, + "sando": 35921, + "sandoval": 44157, + "sandra": 33733, + "sandra": 13415, + "sandro": 42389, + "sands": 5936, + "sandstone": 36796, + "sandwich": 17050, + "sandwich": 8687, + "sandwiches": 19667, + "sandy": 29679, + "sandy": 10355, + "sane": 23419, + "sanford": 32330, + "sanfrancisco": 20254, + "sang": 13235, + "sang": 11684, + "sange": 12466, + "sangria": 42665, + "sani": 39137, + "sani": 34492, + "sanitary": 33842, + "sanitation": 25414, + "saniti": 43987, + "sanity": 30517, + "sanjay": 31712, + "sanjay": 25796, + "sanje": 40405, + "sanjose": 45971, + "sank": 43692, + "sano": 34053, + "sans": 16982, + "sansk": 39689, + "sanskrit": 48083, + "sant": 8356, + "sant": 23120, + "santa": 22175, + "santa": 4555, + "santac": 28876, + "santam": 45627, + "santana": 27033, + "santander": 46476, + "santi": 13856, + "santiago": 16568, + "santo": 29631, + "santo": 18400, + "santor": 28448, + "santorini": 39573, + "santos": 16582, + "sany": 47679, + "sao": 28026, + "sap": 8089, + "sap": 11591, + "sapi": 40016, + "sapp": 13427, + "sapp": 40729, + "sapphire": 22044, + "sar": 1808, + "sar": 9424, + "sara": 37196, + "sara": 10063, + "sarab": 40716, + "sarac": 35722, + "sarah": 9086, + "sarah": 5327, + "saraj": 42592, + "sarajevo": 48211, + "saras": 20373, + "sarasota": 31990, + "sarato": 24845, + "saratoga": 29496, + "sarawak": 47331, + "sarcasm": 37246, + "sarcastic": 48639, + "sardar": 41786, + "sarde": 43925, + "sardin": 27383, + "sardinia": 41025, + "sare": 13051, + "saree": 30860, + "sargent": 34864, + "sari": 42327, + "sari": 20261, + "saries": 47586, + "sarkar": 30673, + "sarko": 33658, + "sarkodie": 42848, + "sarmy": 20954, + "sart": 33006, + "sary": 15398, + "sas": 3960, + "sas": 5235, + "sash": 35656, + "sasha": 46078, + "sasha": 20894, + "sasia": 44751, + "sask": 47091, + "sask": 30416, + "saskat": 17102, + "saskatchewan": 23899, + "saskatoon": 31128, + "sass": 31351, + "sassy": 20827, + "sat": 1382, + "sat": 3279, + "sata": 41520, + "satan": 19446, + "satanic": 38224, + "satchel": 45908, + "sate": 35749, + "satell": 9031, + "satellite": 10316, + "satellites": 28483, + "sath": 29675, + "sathletics": 30154, + "sati": 7038, + "satin": 21803, + "sation": 23674, + "sations": 31232, + "satire": 29875, + "satis": 9906, + "satisf": 22941, + "satisfaction": 19925, + "satisfied": 18101, + "satisfy": 29444, + "satisfying": 23755, + "sato": 34376, + "satu": 45283, + "satur": 1634, + "saturated": 32466, + "saturday": 12537, + "saturday": 1748, + "saturdaymorning": 29053, + "saturdaymotivation": 40843, + "saturdays": 18930, + "saturn": 17312, + "saty": 39426, + "sau": 2096, + "sau": 19455, + "sauce": 5520, + "saucer": 42272, + "sauces": 40367, + "saucy": 46684, + "saudi": 24511, + "saudi": 8548, + "saudiarabia": 28680, + "sauer": 46333, + "saul": 47623, + "saul": 23252, + "sault": 40361, + "sauna": 35460, + "saunders": 23794, + "saur": 13227, + "saura": 46532, + "saurus": 22118, + "saus": 36121, + "sausage": 11855, + "sausages": 31593, + "sauté": 36290, + "sautéed": 38517, + "sauvi": 30116, + "sauvignon": 32745, + "sav": 2248, + "sav": 26533, + "sava": 40198, + "savag": 43039, + "savage": 11859, + "savannah": 18662, + "save": 5895, + "save": 2673, + "saved": 7137, + "saveour": 33390, + "saver": 20987, + "savers": 31416, + "saves": 12907, + "savethe": 18031, + "savi": 14721, + "saving": 28498, + "saving": 6979, + "savings": 10651, + "savior": 24762, + "saviour": 35800, + "savor": 48071, + "savory": 32992, + "savoury": 49071, + "savoy": 39552, + "savvy": 29278, + "saw": 12429, + "saw": 2425, + "sawa": 39613, + "sawards": 29012, + "sawyer": 27726, + "sax": 14169, + "sax": 23766, + "saxon": 31856, + "saxophon": 43760, + "saxophone": 32296, + "say": 3047, + "say": 1451, + "saya": 35170, + "sayang": 46322, + "sayers": 44116, + "sayin": 23662, + "saying": 4455, + "says": 1563, + "saz": 35577, + "sb": 5576, + "sb": 4977, + "sba": 44970, + "sback": 43840, + "sband": 27539, + "sbaseball": 46491, + "sbball": 39190, + "sbc": 31404, + "sberg": 20358, + "sbi": 41369, + "sbk": 39211, + "sboro": 18909, + "sbridge": 49228, + "sbs": 18883, + "sbu": 48075, + "sbu": 46281, + "sburg": 7390, + "sburgh": 48205, + "sbury": 14081, + "sby": 26519, + "sby": 10287, + "sc": 663, + "sc": 3219, + "sca": 11001, + "scab": 31716, + "scaf": 28981, + "scafe": 45574, + "scaffolding": 41687, + "scal": 10859, + "scala": 37997, + "scalable": 44084, + "scale": 37817, + "scale": 5879, + "scaled": 41923, + "scales": 22891, + "scaling": 29116, + "scallo": 19936, + "scallop": 39544, + "scallops": 31430, + "scalp": 38898, + "scam": 17620, + "scam": 13215, + "scamp": 28451, + "scams": 34395, + "scan": 10650, + "scan": 11261, + "scanada": 27121, + "scand": 8110, + "scandal": 35420, + "scandal": 11622, + "scandals": 45490, + "scandin": 32014, + "scandinavian": 35661, + "scanned": 43719, + "scanner": 24185, + "scanning": 24092, + "scans": 31251, + "scap": 35883, + "scape": 36005, + "scape": 12314, + "scapes": 31933, + "scar": 4171, + "scar": 18088, + "scarborough": 24254, + "scarce": 38572, + "scarcity": 45812, + "scare": 33536, + "scare": 15920, + "scarec": 38814, + "scarecrow": 46504, + "scared": 9870, + "scares": 34096, + "scarf": 13365, + "scari": 27050, + "scariest": 37213, + "scarlet": 20389, + "scarlett": 28325, + "scars": 20747, + "scarves": 29249, + "scary": 9250, + "scat": 13899, + "scattered": 22090, + "scavenger": 36778, + "scc": 19458, + "scd": 48422, + "scen": 2204, + "scenario": 20456, + "scenarios": 31346, + "scence": 33418, + "scene": 3562, + "scenery": 16025, + "scenes": 5415, + "scenic": 15394, + "scent": 36277, + "scent": 7683, + "scented": 27190, + "scenter": 23059, + "scentre": 39371, + "scents": 26336, + "scep": 24439, + "scfc": 38578, + "sch": 844, + "sch": 7542, + "scha": 42809, + "schaf": 45588, + "schaft": 41010, + "schal": 35568, + "schalke": 41029, + "schallenge": 43665, + "schan": 31328, + "schar": 15085, + "schat": 31842, + "schau": 35830, + "sche": 3038, + "sche": 7289, + "schedu": 4207, + "schedule": 5521, + "scheduled": 10986, + "schedules": 28986, + "scheduling": 32216, + "scheer": 26776, + "schel": 39881, + "schel": 38569, + "schem": 17720, + "scheme": 9024, + "schemes": 22958, + "schen": 22738, + "scher": 21925, + "scher": 21299, + "schi": 13731, + "schi": 24984, + "schicago": 46230, + "schiff": 39431, + "schild": 32148, + "schiz": 33230, + "schizoph": 40004, + "schizophre": 41163, + "schle": 32022, + "schmid": 17375, + "schmidt": 18463, + "schnau": 45745, + "schnei": 19941, + "schneider": 22972, + "schnit": 40903, + "scho": 2493, + "schoice": 23860, + "schol": 4498, + "scholar": 7192, + "scholar": 12830, + "scholarly": 41065, + "scholars": 13818, + "scholarship": 9070, + "scholarships": 17866, + "scholastic": 35743, + "schoo": 20721, + "school": 6063, + "school": 1228, + "schooled": 44722, + "schoolers": 31455, + "schooling": 28608, + "schools": 3513, + "schre": 47685, + "schri": 25453, + "schro": 32381, + "schu": 11318, + "schubert": 46939, + "schul": 14945, + "schultz": 30308, + "schulz": 39572, + "schumacher": 39208, + "schumer": 25313, + "schur": 42475, + "schwab": 47602, + "schwar": 13985, + "schwartz": 30617, + "schwarz": 27074, + "schwarzenegger": 33860, + "schwe": 25324, + "sci": 2267, + "sci": 8309, + "sciart": 31704, + "scicom": 28606, + "scicomm": 29573, + "scien": 39261, + "science": 10201, + "science": 2497, + "sciencefiction": 39170, + "sciences": 11481, + "scienti": 4338, + "scientific": 9750, + "scientist": 11083, + "scientists": 8045, + "sciento": 36193, + "scientology": 44694, + "scifi": 41862, + "scifi": 12230, + "scion": 47208, + "sciss": 25667, + "scissors": 30867, + "sciutto": 44392, + "sclerosis": 39446, + "sclub": 20017, + "sco": 1065, + "sco": 4763, + "scoe": 31164, + "scol": 13599, + "scoll": 44895, + "scollege": 39536, + "scom": 26407, + "scon": 17163, + "scon": 29272, + "scones": 36443, + "sconf": 39704, + "scoo": 14199, + "scooby": 34469, + "scoop": 13829, + "scoops": 41360, + "scope": 7979, + "scopes": 30328, + "scopic": 23869, + "scopy": 20018, + "scor": 8442, + "score": 12067, + "score": 4431, + "scoreboard": 30104, + "scorecard": 38128, + "scored": 6143, + "scoreless": 33469, + "scorer": 16572, + "scorers": 26699, + "scores": 7039, + "scoring": 9198, + "scorpi": 15445, + "scorpio": 34331, + "scorpion": 28461, + "scorpions": 45401, + "scorsese": 45975, + "scot": 2496, + "scot": 9271, + "scotch": 16687, + "scoti": 46446, + "scotia": 27859, + "scotland": 29174, + "scotland": 4203, + "scots": 17260, + "scotsman": 39612, + "scott": 7775, + "scott": 3664, + "scotti": 6227, + "scottish": 18039, + "scottish": 7442, + "scottsdale": 27817, + "scotty": 39697, + "scotty": 26836, + "scotus": 21720, + "scou": 44909, + "scoun": 16110, + "scouncil": 48787, + "scountry": 40432, + "scour": 46172, + "scout": 32213, + "scout": 10786, + "scouting": 19072, + "scouts": 14837, + "scow": 27929, + "scowboys": 31386, + "scp": 45030, + "scr": 36131, + "scra": 11187, + "scrabble": 39488, + "scram": 17289, + "scramble": 32688, + "scrambled": 39026, + "scran": 41774, + "scranton": 45274, + "scrap": 27950, + "scrap": 21695, + "scrapbook": 48733, + "scrapped": 43325, + "scraps": 40809, + "scrat": 9572, + "scratch": 13258, + "scratched": 48831, + "scratches": 46556, + "scratching": 44617, + "scre": 1795, + "scream": 31645, + "scream": 13239, + "screamed": 35427, + "screaming": 12891, + "screams": 23989, + "screen": 5351, + "screen": 3750, + "screened": 31450, + "screening": 6688, + "screenings": 27655, + "screenplay": 30058, + "screens": 12689, + "screenshot": 20637, + "screenshot": 12646, + "screenshots": 26783, + "screenshotsaturday": 21406, + "screenwriter": 37293, + "screenwriting": 35465, + "screw": 25529, + "screw": 14225, + "screwdriver": 48748, + "screwed": 30592, + "screws": 38292, + "scri": 2139, + "scrib": 34259, + "scribe": 36228, + "scribed": 38334, + "scricket": 45947, + "scrim": 21978, + "scrimmage": 25216, + "scrip": 11955, + "script": 8374, + "scripted": 40513, + "scription": 26604, + "scriptions": 39512, + "scripts": 20109, + "scripture": 27186, + "scro": 30768, + "scroll": 24160, + "scrolling": 28889, + "scrolls": 38113, + "scroo": 42263, + "scru": 7589, + "scrub": 23432, + "scrubs": 37919, + "scrum": 29047, + "scrump": 39791, + "scrumptious": 40987, + "scrutiny": 34305, + "scs": 26853, + "sct": 39284, + "scu": 8181, + "scu": 32135, + "scuba": 39053, + "scuba": 20559, + "scubadiving": 49046, + "scue": 25955, + "scul": 4948, + "scully": 36598, + "sculp": 6093, + "sculpt": 45044, + "sculpted": 41296, + "sculpting": 44389, + "sculptor": 29409, + "sculpture": 8757, + "sculptures": 20378, + "scum": 29655, + "scumb": 44525, + "scup": 21506, + "scur": 32742, + "scwx": 41966, + "scy": 27471, + "sd": 3080, + "sd": 4159, + "sda": 25548, + "sdale": 12327, + "sday": 5902, + "sday": 1376, + "sdays": 14491, + "sdc": 40992, + "sdcc": 13246, + "sden": 17241, + "sdf": 34681, + "sdg": 20177, + "sdgs": 16261, + "sdk": 40015, + "sdlive": 34561, + "sdn": 41925, + "sdsu": 41284, + "se": 567, + "se": 611, + "sea": 5970, + "sea": 2102, + "seab": 15728, + "seabir": 42558, + "seac": 35626, + "seaf": 9336, + "seafood": 12472, + "seag": 15730, + "seagu": 38076, + "seagull": 38858, + "seagulls": 42215, + "seahawks": 15341, + "seal": 21381, + "seal": 10159, + "sealed": 13358, + "sealing": 42992, + "seals": 18179, + "seam": 13710, + "seam": 44201, + "seaman": 47513, + "seamless": 29373, + "seamus": 40175, + "sean": 11406, + "sean": 6077, + "seanhannity": 43316, + "seap": 29983, + "seaport": 46418, + "sear": 1612, + "search": 23129, + "search": 1920, + "searched": 28961, + "searches": 26378, + "searching": 10626, + "seared": 29727, + "sears": 26693, + "seas": 7329, + "seas": 9556, + "seascape": 42593, + "seaside": 18867, + "season": 19288, + "season": 1367, + "seasonal": 14215, + "seasoned": 28399, + "seasoning": 43439, + "seasons": 8635, + "seat": 19670, + "seat": 4922, + "seated": 23953, + "seater": 37543, + "seating": 16240, + "seats": 6944, + "seattle": 24388, + "seattle": 6274, + "seau": 32263, + "seaw": 32658, + "seaweed": 30204, + "seaworld": 27422, + "seb": 35766, + "seb": 25171, + "sebasti": 10324, + "sebastian": 43792, + "sebastian": 13181, + "sebring": 41086, + "sec": 2875, + "sec": 5338, + "seca": 37847, + "secco": 27394, + "sece": 46297, + "seclu": 42392, + "secon": 1846, + "second": 9329, + "second": 2241, + "secondary": 13107, + "seconds": 6541, + "secre": 2460, + "secret": 20710, + "secret": 4145, + "secretari": 29515, + "secretariat": 31767, + "secretary": 6552, + "secretly": 21400, + "secrets": 9735, + "secs": 28665, + "sect": 15772, + "section": 34986, + "section": 4853, + "sectional": 21876, + "sections": 20061, + "sector": 6579, + "sectors": 22173, + "secu": 4894, + "secular": 47483, + "secular": 27560, + "secur": 2557, + "secure": 44763, + "secure": 7515, + "secured": 16848, + "secures": 31567, + "securing": 24759, + "securities": 25080, + "security": 31245, + "security": 2741, + "sed": 14034, + "sed": 1252, + "sedan": 24237, + "sedg": 46926, + "sedge": 45288, + "sedi": 29269, + "sedly": 31771, + "sedona": 46862, + "seduc": 19933, + "seductive": 43721, + "see": 1751, + "see": 862, + "seed": 14064, + "seed": 6488, + "seeded": 33688, + "seeding": 40050, + "seedlings": 47933, + "seeds": 9128, + "seeing": 3214, + "seek": 8839, + "seeker": 28011, + "seekers": 20732, + "seeking": 8592, + "seeks": 12594, + "seem": 20043, + "seem": 7523, + "seemed": 17240, + "seemingly": 25917, + "seems": 4453, + "seen": 36273, + "seen": 2041, + "seer": 32486, + "sees": 7594, + "seeyou": 41279, + "sef": 27453, + "seg": 10551, + "sega": 16122, + "segment": 15615, + "segments": 43053, + "segreg": 49117, + "segregation": 39086, + "segu": 33156, + "segun": 43087, + "seh": 27536, + "seh": 41430, + "sehun": 17705, + "sei": 13130, + "sei": 15907, + "sein": 24669, + "seine": 41378, + "seinfeld": 33706, + "seis": 25559, + "seismic": 38459, + "seiz": 22171, + "seize": 26624, + "seized": 15826, + "seizure": 36804, + "seizures": 47199, + "sek": 45515, + "sek": 25880, + "sel": 1000, + "sel": 4098, + "sela": 47006, + "selamat": 37692, + "selangor": 44402, + "selby": 43546, + "selca": 38606, + "selcaday": 35924, + "seldom": 48322, + "sele": 29137, + "selec": 3014, + "select": 8690, + "selected": 6881, + "selecting": 32696, + "selection": 6724, + "selections": 24099, + "selective": 28686, + "selects": 32902, + "selen": 19970, + "selena": 14677, + "selenagomez": 27653, + "seley": 30556, + "self": 10139, + "self": 1322, + "selfcare": 39560, + "selfi": 3007, + "selfie": 26735, + "selfie": 3666, + "selfies": 46058, + "selfies": 10050, + "selfish": 26907, + "selfless": 34236, + "sell": 10279, + "sell": 5119, + "seller": 11779, + "sellers": 16562, + "selling": 4396, + "sells": 14306, + "selma": 36652, + "sels": 42070, + "selves": 4505, + "sely": 8402, + "sem": 8645, + "sem": 17106, + "sema": 31816, + "seman": 29119, + "seman": 28378, + "semana": 41780, + "semb": 36054, + "seme": 10855, + "sement": 10714, + "sements": 31449, + "semester": 11905, + "semi": 11023, + "semi": 6684, + "semic": 26967, + "semicon": 34315, + "semiconduc": 35646, + "semiconductor": 43551, + "semifinal": 22935, + "semifinals": 21863, + "semin": 5595, + "seminar": 7269, + "seminars": 34870, + "seminary": 31655, + "seminole": 42956, + "semis": 24013, + "semit": 22628, + "semite": 23721, + "semitic": 34894, + "semitism": 25911, + "semper": 47391, + "sen": 1057, + "sen": 2249, + "sena": 21584, + "senate": 30703, + "senate": 6843, + "senator": 20871, + "senator": 8495, + "senators": 16889, + "send": 27684, + "send": 3625, + "sending": 6985, + "sends": 10817, + "sene": 25269, + "seneca": 33419, + "senegal": 28255, + "senew": 49313, + "seng": 43022, + "seng": 29971, + "senior": 19865, + "senior": 3415, + "seniors": 8138, + "senna": 36195, + "senpai": 46562, + "sens": 5218, + "sens": 22837, + "sensation": 19383, + "sensational": 23051, + "sense": 29162, + "sense": 4747, + "sensei": 36158, + "senses": 21809, + "sensi": 38802, + "sensible": 30635, + "sensing": 29236, + "sensiti": 20531, + "sensitive": 13734, + "sensitivity": 27788, + "sensor": 15330, + "sensors": 20356, + "sensory": 21831, + "sensu": 28157, + "sensual": 40860, + "sent": 6200, + "sent": 3676, + "sentence": 12737, + "sentenced": 17773, + "sentences": 25858, + "sentencing": 34394, + "senti": 19042, + "sentim": 25102, + "sentiment": 25949, + "sentimental": 40070, + "sentiments": 47450, + "sentin": 20042, + "sentinel": 23123, + "senting": 3924, + "seo": 24743, + "seo": 8622, + "seok": 34697, + "seok": 22482, + "seokjin": 45584, + "seoul": 13253, + "sep": 3212, + "sep": 10434, + "separ": 6859, + "separate": 13886, + "separated": 22163, + "separately": 41904, + "separates": 45365, + "separati": 39377, + "separating": 43480, + "separation": 22007, + "sephora": 38414, + "sepsis": 40205, + "sept": 5380, + "septe": 3672, + "september": 3707, + "septic": 34690, + "sepul": 47360, + "seq": 44379, + "sequ": 5491, + "seque": 44662, + "sequel": 15701, + "sequence": 18833, + "sequences": 47306, + "sequencing": 33484, + "sequo": 32781, + "sequoia": 42404, + "ser": 803, + "ser": 2771, + "sera": 28250, + "serbia": 19038, + "serbian": 33687, + "sere": 35770, + "seren": 7880, + "serena": 19519, + "serenawilliams": 48316, + "serendip": 45805, + "serendipity": 49386, + "serene": 28269, + "serenity": 24187, + "serge": 13477, + "serge": 35700, + "sergeant": 22049, + "sergei": 39870, + "sergey": 35390, + "sergi": 47675, + "sergio": 18359, + "seri": 2763, + "seri": 37509, + "serial": 14216, + "serie": 19752, + "seriea": 32660, + "series": 1857, + "serious": 47421, + "serious": 4770, + "seriously": 4885, + "sermon": 24884, + "sero": 48883, + "serpent": 37084, + "serpent": 35364, + "serra": 39851, + "serrano": 44236, + "sers": 13509, + "serum": 25385, + "serv": 1297, + "serv": 24571, + "servant": 20810, + "servants": 29652, + "serve": 39202, + "serve": 2838, + "served": 4740, + "server": 36458, + "server": 8398, + "serverless": 49243, + "servers": 22262, + "serves": 9915, + "servic": 27115, + "service": 21496, + "service": 2086, + "serviced": 44687, + "services": 3100, + "servicing": 41300, + "serving": 5722, + "sery": 14279, + "ses": 23708, + "ses": 1386, + "sesame": 21706, + "sese": 37128, + "sesh": 24274, + "session": 2550, + "sessions": 6327, + "set": 7965, + "set": 1167, + "setback": 43605, + "seth": 20005, + "seth": 11870, + "sethu": 38933, + "setlist": 33141, + "seton": 43799, + "sets": 4650, + "sett": 4984, + "sett": 17567, + "sette": 14613, + "setter": 23153, + "settes": 44145, + "setti": 45170, + "setting": 5264, + "settings": 18628, + "settle": 15075, + "settled": 18310, + "settlement": 16494, + "settlements": 36605, + "settlers": 35671, + "settles": 41498, + "settling": 22036, + "setup": 11092, + "seu": 31539, + "seul": 48975, + "seum": 18838, + "seun": 24209, + "seung": 32393, + "seung": 33711, + "seungri": 41627, + "seuss": 34441, + "sev": 26585, + "sev": 37600, + "seva": 42604, + "seve": 21458, + "seve": 22468, + "sevel": 17439, + "seven": 7874, + "seven": 5757, + "sevens": 29911, + "sevent": 43048, + "seventeen": 19337, + "seventh": 17568, + "seventy": 47170, + "sever": 3250, + "sever": 45557, + "several": 5560, + "severance": 26194, + "severe": 6215, + "severely": 24417, + "severn": 34626, + "severy": 34207, + "sevilla": 24947, + "seville": 34988, + "sew": 28640, + "sewage": 32777, + "sewer": 28294, + "sewing": 15974, + "sewn": 42118, + "sex": 3548, + "sex": 5937, + "sexi": 20562, + "sexiest": 25426, + "sexism": 32059, + "sexist": 33047, + "sexu": 14741, + "sexual": 6749, + "sexuality": 21244, + "sexually": 23032, + "sexy": 21019, + "sexy": 38127, + "sey": 6317, + "sey": 2258, + "seychel": 36809, + "seychelles": 38519, + "seye": 35604, + "seym": 22657, + "seymour": 25850, + "seys": 15081, + "sez": 42377, + "señ": 43368, + "sf": 4435, + "sf": 4915, + "sfa": 32675, + "sfam": 37649, + "sfb": 27930, + "sfc": 14129, + "sfest": 49024, + "sff": 42056, + "sfgiants": 20923, + "sfield": 11801, + "sfo": 39182, + "sfootball": 45259, + "sfor": 9115, + "sford": 28917, + "sforsale": 28888, + "sfw": 18073, + "sfx": 37995, + "sg": 9599, + "sg": 7611, + "sga": 33049, + "sgate": 27558, + "sgh": 47590, + "sgo": 5393, + "sgo": 21044, + "sgt": 13748, + "sh": 552, + "sh": 849, + "sha": 1514, + "sha": 3337, + "shaa": 44221, + "shab": 8323, + "shabbat": 38042, + "shabby": 28838, + "shack": 23866, + "shack": 18785, + "shad": 3182, + "shad": 23874, + "shade": 34554, + "shade": 10097, + "shaded": 43506, + "shades": 46608, + "shades": 9270, + "shadesof": 45180, + "shading": 37348, + "shado": 9325, + "shadow": 15243, + "shadow": 7068, + "shadowhun": 19931, + "shadowhunters": 24834, + "shadowing": 46092, + "shadows": 12971, + "shady": 22158, + "shaf": 12032, + "shaft": 21545, + "shag": 22439, + "shaggy": 42662, + "shah": 13203, + "shah": 8439, + "shahe": 23643, + "shaheed": 30060, + "shaheer": 43969, + "shahi": 46972, + "shahid": 25696, + "shahid": 27138, + "shahidkapoor": 29892, + "shahzad": 45915, + "shai": 47941, + "shaikh": 45712, + "shail": 37603, + "shair": 43135, + "shak": 8385, + "shake": 8206, + "shake": 8251, + "shaken": 38237, + "shaker": 26210, + "shakers": 38411, + "shakes": 19668, + "shakespe": 9890, + "shakespeare": 22499, + "shakespeare": 12488, + "shakespearesunday": 32320, + "shaking": 19101, + "shakira": 40795, + "shakti": 48593, + "shakti": 32458, + "shakur": 48915, + "shal": 15056, + "shal": 28175, + "shale": 32864, + "shall": 4742, + "shallow": 23730, + "shalom": 31339, + "sham": 6453, + "sham": 9005, + "shaman": 48727, + "shambles": 40799, + "shame": 14776, + "shame": 7593, + "shameful": 28283, + "shameless": 25380, + "shaming": 40553, + "shampoo": 23944, + "shamrock": 34199, + "shan": 5171, + "shan": 8834, + "shana": 44835, + "shand": 29101, + "shane": 26863, + "shane": 11572, + "shang": 11141, + "shanghai": 12742, + "shani": 46665, + "shank": 24685, + "shankar": 24108, + "shann": 9932, + "shannon": 22842, + "shannon": 13581, + "shant": 36610, + "shap": 5581, + "shape": 26925, + "shape": 6448, + "shaped": 10127, + "shapes": 15377, + "shaping": 18632, + "shapiro": 32110, + "shaq": 46402, + "shaq": 26843, + "shar": 1669, + "shar": 36542, + "shara": 48849, + "sharapo": 36489, + "sharapova": 36671, + "shard": 42207, + "share": 7585, + "share": 1978, + "shared": 5368, + "shareholder": 38241, + "shareholders": 34778, + "sharepoint": 39213, + "shares": 4974, + "sharethe": 49277, + "shareyour": 45890, + "shari": 27738, + "shari": 47390, + "sharia": 37244, + "sharif": 15501, + "sharing": 3567, + "sharjah": 33420, + "shark": 15836, + "shark": 7980, + "sharks": 10047, + "sharkweek": 39571, + "sharma": 10105, + "sharon": 28722, + "sharon": 14138, + "sharp": 17126, + "sharp": 8157, + "sharpe": 34374, + "sharpen": 41465, + "sharpie": 46858, + "sharply": 37185, + "shasta": 46727, + "shat": 12169, + "shat": 44388, + "shatter": 45008, + "shattered": 26820, + "shau": 13750, + "shaun": 23446, + "shaun": 16669, + "shav": 11410, + "shave": 17735, + "shaved": 25571, + "shaving": 24261, + "shaw": 6122, + "shaw": 6805, + "shawa": 46413, + "shawl": 35132, + "shawn": 16677, + "shawn": 10970, + "shawnee": 48060, + "shawnmendes": 27277, + "shawty": 38026, + "shay": 10778, + "shay": 18361, + "shaykh": 47223, + "shaz": 18618, + "shazam": 29063, + "shc": 43419, + "shd": 37729, + "she": 1729, + "she": 1043, + "shea": 20407, + "shead": 44287, + "shead": 20434, + "shealth": 41743, + "shealth": 22197, + "shear": 27974, + "shear": 32108, + "shearer": 40505, + "sheath": 45637, + "shed": 16586, + "shed": 1492, + "shedding": 33608, + "sheds": 25921, + "shee": 23450, + "shee": 34321, + "sheed": 26105, + "sheehan": 41809, + "sheen": 25025, + "sheep": 23604, + "sheep": 9629, + "sheer": 17577, + "sheeran": 18561, + "sheet": 7298, + "sheets": 12744, + "shef": 8237, + "sheff": 38844, + "sheff": 43821, + "sheffiel": 26940, + "sheffield": 41763, + "sheffield": 10420, + "sheffieldissuper": 33628, + "sheh": 31667, + "sheikh": 15031, + "sheil": 42765, + "sheila": 25734, + "shek": 33285, + "shel": 3159, + "shelby": 36906, + "shelby": 16885, + "sheldon": 25079, + "shelf": 10955, + "shell": 23374, + "shell": 6648, + "shelley": 22497, + "shelling": 43166, + "shells": 19265, + "shelly": 37461, + "shelter": 8599, + "sheltered": 48070, + "shelters": 24312, + "shelton": 24471, + "shelves": 16225, + "shem": 40299, + "shen": 10154, + "shen": 31098, + "shenan": 20965, + "shenando": 44666, + "shenanigans": 26590, + "shenko": 39751, + "shenmue": 48279, + "shenzhen": 38970, + "shep": 33757, + "shep": 44857, + "shepard": 26810, + "shepher": 11008, + "shepherd": 13242, + "shepherds": 42792, + "sheppard": 37304, + "sher": 3570, + "sher": 4510, + "sheraton": 39400, + "shere": 21507, + "sheri": 9235, + "sheridan": 27085, + "sheriff": 10309, + "sherlock": 17294, + "sherman": 17822, + "sherry": 44348, + "sherry": 24689, + "shers": 14141, + "sherwood": 24527, + "sheryl": 39773, + "shes": 45514, + "shes": 2502, + "shet": 15850, + "shetland": 29595, + "shetty": 25533, + "shev": 45182, + "sheva": 45132, + "shh": 35025, + "shhh": 36932, + "shi": 823, + "shi": 3533, + "shia": 23791, + "shibu": 36177, + "shibuya": 41623, + "shie": 26638, + "shiel": 33413, + "shield": 8670, + "shields": 19085, + "shies": 35312, + "shif": 35317, + "shift": 43767, + "shift": 6905, + "shifted": 34429, + "shifter": 48944, + "shifting": 21992, + "shifts": 23957, + "shik": 36980, + "shil": 14370, + "shill": 32121, + "shill": 30090, + "shilpa": 47062, + "shilpa": 40690, + "shim": 11986, + "shim": 32780, + "shima": 14382, + "shimano": 48904, + "shimi": 40517, + "shimmer": 38792, + "shin": 5664, + "shin": 11784, + "shinde": 41516, + "shine": 17582, + "shine": 3780, + "shinee": 19660, + "shines": 16015, + "shing": 38641, + "shing": 1743, + "shining": 10485, + "shino": 43074, + "shiny": 12190, + "ship": 7645, + "ship": 1158, + "shipment": 28553, + "shipp": 34709, + "shipped": 15279, + "shippers": 44789, + "shipping": 5721, + "ships": 3262, + "shipwreck": 48878, + "shipy": 26828, + "shipyard": 31273, + "shir": 1956, + "shiraz": 35618, + "shire": 11975, + "shire": 2968, + "shirehour": 32456, + "shirley": 18189, + "shiro": 26048, + "shirt": 27576, + "shirt": 2523, + "shirtless": 28959, + "shirts": 5803, + "shistory": 34979, + "shiv": 18042, + "shiv": 37121, + "shiva": 33881, + "shiva": 21174, + "shka": 38944, + "shld": 49359, + "shma": 48074, + "shment": 8802, + "shments": 18822, + "sho": 719, + "sho": 13756, + "shock": 19617, + "shock": 8736, + "shocked": 15787, + "shocker": 37971, + "shockey": 22258, + "shocking": 13394, + "shocks": 31886, + "shoe": 16308, + "shoe": 7342, + "shoes": 49391, + "shoes": 4079, + "shol": 21472, + "sholm": 44139, + "shome": 42701, + "shon": 19526, + "shon": 37621, + "shone": 47173, + "shoo": 1975, + "shook": 20730, + "shoops": 29956, + "shoot": 12531, + "shoot": 3704, + "shooter": 13645, + "shooters": 31902, + "shooting": 3992, + "shootings": 26753, + "shootout": 20666, + "shoots": 14144, + "shop": 5738, + "shop": 1557, + "shopify": 47949, + "shoplocal": 21775, + "shopp": 38486, + "shoppe": 38236, + "shopped": 28088, + "shopper": 24346, + "shoppers": 22316, + "shopping": 42101, + "shopping": 4266, + "shops": 6467, + "shopsmall": 35942, + "shor": 3209, + "shore": 14717, + "shore": 5928, + "shored": 33140, + "shoreditch": 35042, + "shoreline": 34807, + "shores": 18102, + "short": 6803, + "short": 3005, + "shortage": 19910, + "shortages": 38730, + "shortcuts": 45793, + "shorten": 41711, + "shorter": 20350, + "shortest": 33717, + "shortfilm": 37204, + "shorth": 37397, + "shortlist": 28163, + "shortlisted": 20631, + "shortly": 11967, + "shorts": 9680, + "shorty": 33502, + "shot": 9805, + "shot": 2000, + "shotel": 42365, + "shotgun": 21643, + "shots": 5342, + "shou": 3890, + "shoul": 29847, + "should": 14947, + "should": 1535, + "shoulder": 8476, + "shoulders": 18738, + "shouldn": 9416, + "shour": 20025, + "shouse": 28671, + "shout": 7335, + "shout": 5214, + "shouted": 44397, + "shouting": 26464, + "shoutout": 8274, + "shouts": 26709, + "shovel": 31778, + "show": 2133, + "show": 1080, + "showbiz": 34156, + "showcas": 14290, + "showcase": 7265, + "showcased": 35786, + "showcases": 26266, + "showcasing": 17036, + "showdown": 15576, + "showed": 7150, + "shower": 7777, + "showers": 9893, + "showing": 3649, + "shown": 8506, + "showroom": 16821, + "shows": 2665, + "showtime": 40576, + "showtime": 15442, + "showyour": 46733, + "shp": 38341, + "shq": 21145, + "shr": 10118, + "shra": 21360, + "shradd": 28172, + "shraddha": 35208, + "shraddhakapoor": 40385, + "shre": 12101, + "shred": 19756, + "shred": 33017, + "shredded": 31772, + "shredding": 45534, + "shree": 37410, + "shrek": 35009, + "shrews": 26411, + "shrewsbury": 30921, + "shri": 8838, + "shri": 11424, + "shrimp": 12727, + "shrin": 24865, + "shrine": 16156, + "shrink": 34957, + "shrinking": 41243, + "shrm": 44163, + "shro": 15259, + "shroff": 32081, + "shrop": 22630, + "shropshire": 26344, + "shru": 14911, + "shrub": 41464, + "shrubs": 47975, + "shrun": 46767, + "shs": 16184, + "sht": 44210, + "shti": 38927, + "shu": 2872, + "shu": 17651, + "shua": 33771, + "shub": 40552, + "shud": 45782, + "shuff": 42641, + "shuffle": 21681, + "shui": 45473, + "shuk": 29927, + "shukla": 46829, + "shul": 30721, + "shum": 37383, + "shun": 24479, + "shun": 39594, + "shur": 41032, + "shut": 8702, + "shut": 8282, + "shutdown": 16051, + "shutout": 24385, + "shuts": 28313, + "shutt": 31866, + "shutter": 36235, + "shutter": 33902, + "shutters": 46894, + "shutting": 31383, + "shuttle": 15842, + "shwar": 41640, + "shy": 22678, + "shy": 9682, + "si": 564, + "si": 2990, + "sia": 2357, + "siam": 29686, + "siam": 48248, + "siamese": 43161, + "sian": 28510, + "sian": 6221, + "sians": 26583, + "sias": 28645, + "siber": 22206, + "siberia": 39969, + "siberian": 34058, + "sibl": 14338, + "sible": 14507, + "sibling": 43060, + "sibling": 23779, + "siblings": 17156, + "sic": 8278, + "sic": 1118, + "sica": 34125, + "sical": 33875, + "sichuan": 48950, + "sicilian": 45292, + "sicily": 23179, + "sick": 11143, + "sick": 5359, + "sickest": 47972, + "sickle": 41459, + "sickness": 28898, + "sics": 26297, + "sid": 10117, + "sid": 15119, + "sidd": 19842, + "siddi": 35227, + "side": 5869, + "side": 1145, + "sided": 21061, + "sidekick": 44683, + "sidel": 43557, + "sideline": 32056, + "sidelines": 31046, + "sider": 30581, + "siders": 41249, + "sides": 7578, + "sideshow": 46789, + "sidewalk": 23278, + "sidewalks": 43583, + "sideways": 35593, + "siding": 38758, + "sidney": 22598, + "sie": 8533, + "sie": 5685, + "sieg": 49203, + "siege": 18460, + "siegel": 48559, + "siem": 18434, + "siemens": 30147, + "siempre": 44030, + "siena": 33336, + "sienna": 40373, + "sier": 10028, + "sier": 7444, + "sierra": 13552, + "siers": 35923, + "sies": 16367, + "siest": 18323, + "sif": 29300, + "sig": 872, + "sig": 19145, + "sigh": 36303, + "sigh": 15505, + "sighs": 44579, + "sight": 16897, + "sight": 6329, + "sighted": 33034, + "sighting": 17507, + "sightings": 30004, + "sights": 17364, + "sightseeing": 34210, + "sigma": 45075, + "sigma": 15697, + "sign": 5538, + "sign": 2292, + "signage": 21156, + "signal": 10781, + "signaling": 38492, + "signalling": 48426, + "signals": 17150, + "signation": 24347, + "signature": 9189, + "signatures": 21865, + "signed": 3163, + "signee": 39778, + "signi": 34023, + "signific": 6374, + "significance": 23769, + "significant": 8735, + "significantly": 16187, + "signing": 4401, + "signingday": 40282, + "signings": 27731, + "signs": 4659, + "signup": 40791, + "sigue": 49401, + "sii": 36672, + "sik": 19974, + "sik": 22413, + "sika": 31144, + "sikh": 21829, + "sikhs": 45426, + "sil": 1556, + "sil": 8315, + "sila": 41754, + "sile": 37620, + "silen": 39048, + "silence": 8462, + "silenced": 45415, + "silent": 30352, + "silent": 8487, + "silently": 42640, + "silhou": 20589, + "silhouette": 26149, + "silic": 23830, + "silicon": 32412, + "silicon": 17888, + "silicone": 28221, + "silk": 25891, + "silk": 9743, + "silky": 29554, + "sill": 42468, + "sill": 48024, + "silly": 11883, + "silon": 31841, + "sils": 39708, + "silva": 16489, + "silve": 37697, + "silver": 7525, + "silver": 3467, + "silverado": 46160, + "silverstone": 29666, + "silvia": 37289, + "sim": 5026, + "sim": 10740, + "sima": 35871, + "simba": 39492, + "simcoe": 47148, + "sime": 28329, + "simi": 38073, + "simil": 7202, + "similar": 8547, + "similarities": 34716, + "simm": 13001, + "simmons": 14699, + "simo": 37171, + "simon": 8796, + "simon": 6668, + "simona": 46277, + "simone": 19062, + "simons": 33097, + "simp": 2542, + "simple": 19018, + "simple": 4129, + "simpler": 35489, + "simplest": 39588, + "simpli": 16868, + "simplicity": 21262, + "simplified": 36647, + "simplify": 35479, + "simply": 25637, + "simply": 6151, + "simpson": 41805, + "simpson": 11750, + "simpsons": 21092, + "sims": 14021, + "simul": 9845, + "simulated": 46395, + "simulation": 18610, + "simulator": 20821, + "simultaneous": 48816, + "simultaneously": 28575, + "sin": 1303, + "sin": 3421, + "sina": 19541, + "sinai": 33226, + "sinatra": 27262, + "sinc": 30464, + "since": 1855, + "sincere": 24513, + "sincere": 24886, + "sincerely": 25673, + "sinclair": 23100, + "sind": 39598, + "sind": 30877, + "sindh": 20754, + "sindia": 48038, + "sine": 22741, + "sine": 33793, + "sinfo": 47178, + "sing": 1387, + "sing": 1197, + "singapo": 27861, + "singapore": 28879, + "singapore": 6754, + "singer": 33880, + "singer": 5108, + "singers": 15613, + "singersongwriter": 44585, + "singh": 19445, + "singh": 5715, + "singing": 5864, + "single": 19524, + "single": 2688, + "singles": 12025, + "singleton": 46247, + "singly": 16619, + "sings": 13635, + "singul": 34003, + "singular": 44009, + "singularity": 48410, + "sinha": 29416, + "sini": 41781, + "sini": 26319, + "sinister": 31313, + "sink": 37232, + "sink": 14551, + "sinking": 27949, + "sinks": 32710, + "sinn": 36315, + "sinner": 45380, + "sinners": 43436, + "sino": 29759, + "sins": 9345, + "sinthe": 30737, + "sinu": 37351, + "sinus": 47535, + "sio": 10807, + "siob": 40954, + "siology": 46315, + "sion": 5676, + "sion": 1015, + "sional": 14533, + "sionally": 30754, + "sions": 4060, + "sioux": 44695, + "sioux": 24954, + "sip": 16096, + "sipping": 28527, + "sir": 10708, + "sir": 3846, + "sire": 28450, + "siren": 33026, + "sirens": 35907, + "siri": 13986, + "siri": 18394, + "sirius": 23574, + "sirius": 34999, + "siriusxm": 29833, + "sirloin": 46828, + "sis": 18132, + "sis": 2580, + "sisd": 27132, + "sisi": 37892, + "siss": 42929, + "sissy": 27564, + "sist": 20520, + "sista": 37448, + "sister": 17417, + "sister": 3677, + "sisterhood": 37313, + "sisters": 6404, + "sit": 7387, + "sit": 4037, + "sitcom": 30426, + "site": 26792, + "site": 1988, + "sites": 7236, + "sith": 41499, + "sito": 42613, + "sits": 12726, + "sitt": 42988, + "sitter": 40777, + "sittin": 40887, + "sitting": 4919, + "situ": 5562, + "situ": 42536, + "situated": 22030, + "situation": 7144, + "situations": 19096, + "sity": 38177, + "sity": 5477, + "siu": 40174, + "sium": 8090, + "sius": 27595, + "siva": 20991, + "sivan": 36931, + "sive": 23572, + "sive": 1875, + "sively": 10343, + "siveness": 39667, + "sives": 23896, + "sivity": 42738, + "siwon": 29055, + "six": 5968, + "six": 4093, + "sixers": 25941, + "sixteen": 28677, + "sixth": 12909, + "sixties": 44948, + "sixty": 32588, + "siya": 44440, + "size": 38377, + "size": 3235, + "sized": 9832, + "sizes": 10253, + "sizing": 28330, + "sizz": 23778, + "sizzle": 47890, + "sizzling": 35799, + "sj": 7536, + "sj": 16010, + "sjo": 42012, + "sk": 909, + "sk": 2058, + "ska": 7495, + "skag": 31948, + "skan": 46772, + "skar": 27587, + "skar": 26835, + "skate": 13740, + "skate": 12745, + "skateboard": 31777, + "skateboarding": 31352, + "skater": 30337, + "skaters": 39824, + "skates": 31479, + "skc": 44551, + "ske": 6261, + "ske": 25516, + "skel": 36564, + "skelet": 27075, + "skeletal": 37369, + "skeleton": 20062, + "skeletons": 48874, + "skell": 40801, + "skep": 27772, + "skeptical": 44934, + "sker": 37640, + "sker": 33600, + "sket": 3744, + "sketch": 11767, + "sketch": 5269, + "sketchbook": 18899, + "sketched": 38581, + "sketches": 17622, + "sketching": 23228, + "sketchy": 41582, + "skey": 37453, + "ski": 3327, + "ski": 3428, + "skid": 36574, + "skid": 32099, + "skier": 42585, + "skies": 7244, + "skiing": 14400, + "skil": 24543, + "skill": 15598, + "skill": 10604, + "skilled": 17535, + "skillet": 40568, + "skills": 4113, + "skim": 33191, + "skin": 5821, + "skin": 3575, + "skincare": 12648, + "skine": 37300, + "sking": 46215, + "skinned": 42199, + "skinner": 30261, + "skinny": 42729, + "skinny": 15457, + "skins": 11594, + "skip": 39793, + "skip": 14296, + "skipped": 40639, + "skipper": 22226, + "skipping": 34867, + "skir": 8919, + "skirt": 12386, + "skirts": 24840, + "skis": 32843, + "skit": 43573, + "skitchen": 42820, + "skittles": 43213, + "sko": 15141, + "sko": 23493, + "skoda": 38668, + "skool": 26743, + "skril": 43149, + "skrillex": 43651, + "sks": 48136, + "sku": 10836, + "skul": 17561, + "skull": 34068, + "skull": 12092, + "skulls": 31804, + "skunk": 42194, + "sky": 3075, + "sky": 2390, + "skybet": 45540, + "skye": 21475, + "skyl": 43554, + "skylar": 45411, + "skyline": 14606, + "skymap": 41734, + "skynews": 40977, + "skype": 17069, + "skyrim": 33693, + "skysports": 39845, + "skysports": 46725, + "skywalker": 32936, + "sl": 2621, + "sl": 7489, + "sla": 2725, + "sla": 26707, + "slab": 24241, + "slabs": 42818, + "slack": 37108, + "slack": 30142, + "slade": 33546, + "slain": 35972, + "slalom": 43540, + "slam": 14891, + "slam": 10131, + "slammed": 29772, + "slams": 18907, + "slan": 44663, + "slan": 47193, + "sland": 11294, + "slang": 33655, + "slap": 48830, + "slap": 21751, + "slapped": 38861, + "slaps": 46796, + "slash": 19749, + "slat": 38966, + "slate": 17919, + "slated": 36094, + "slater": 25968, + "slaugh": 26782, + "slaughter": 19815, + "slaughtered": 46615, + "slav": 47292, + "slava": 41797, + "slave": 14029, + "slavery": 15754, + "slaves": 23833, + "slaw": 28178, + "slay": 48319, + "slay": 19380, + "slayed": 44870, + "slayer": 21605, + "slaying": 27812, + "slays": 45648, + "slc": 21972, + "sle": 1709, + "sleague": 23336, + "sled": 28438, + "sledge": 48750, + "slee": 17642, + "slee": 38977, + "sleek": 23187, + "sleep": 4656, + "sleep": 3840, + "sleeper": 28709, + "sleeping": 6982, + "sleepless": 39779, + "sleepover": 39415, + "sleeps": 16610, + "sleepy": 32572, + "sleepy": 14497, + "sleet": 36948, + "sleeve": 35270, + "sleeve": 10536, + "sleeveless": 38049, + "sleeves": 19691, + "sleg": 47650, + "sleigh": 30865, + "slender": 40331, + "slept": 20388, + "sler": 14066, + "sley": 17198, + "sley": 6496, + "sli": 1811, + "sli": 44824, + "slic": 19692, + "slice": 13431, + "sliced": 28121, + "slices": 28424, + "slick": 18341, + "slide": 27828, + "slide": 8837, + "slider": 37861, + "sliders": 40700, + "slides": 15939, + "slideshow": 42817, + "sliding": 21468, + "slife": 15448, + "sliga": 21080, + "slight": 14297, + "slightly": 8456, + "sligo": 30424, + "slike": 38744, + "slim": 35226, + "slim": 12364, + "slime": 29107, + "sling": 28021, + "sling": 32607, + "slinger": 47269, + "slions": 43363, + "slip": 39785, + "slip": 12105, + "slipknot": 41816, + "slipped": 30344, + "slipper": 39644, + "slippers": 26509, + "slippery": 30814, + "slipping": 36301, + "slips": 30632, + "slist": 33749, + "slit": 47011, + "slive": 31652, + "slo": 4303, + "slo": 36083, + "sloan": 29110, + "sloane": 41553, + "slogan": 23398, + "slogans": 42795, + "slope": 22769, + "slopes": 24066, + "sloppy": 36154, + "slot": 14500, + "sloth": 30007, + "slots": 19238, + "slou": 48493, + "slovak": 23315, + "slovakia": 25994, + "sloven": 17018, + "slovenia": 21037, + "slow": 6674, + "slow": 5444, + "slowdown": 38421, + "slowed": 43793, + "slower": 29181, + "slowing": 29839, + "slowly": 9568, + "slows": 46855, + "slp": 45599, + "slr": 21325, + "sls": 33651, + "slt": 39283, + "sltd": 36388, + "slu": 7224, + "slu": 47456, + "slug": 34190, + "slugger": 48671, + "slum": 46754, + "slumber": 44295, + "slump": 35588, + "slur": 30476, + "slush": 39815, + "slv": 45526, + "sly": 28145, + "sly": 21062, + "sm": 978, + "sm": 2764, + "sma": 4357, + "sma": 11854, + "smack": 21280, + "smack": 30026, + "smackdown": 26138, + "smafia": 47686, + "smag": 32212, + "smal": 48379, + "small": 5244, + "small": 2442, + "smallbiz": 41724, + "smallbiz": 18987, + "smallbusiness": 21316, + "smalle": 18490, + "smaller": 12431, + "smallest": 18686, + "smalls": 41696, + "sman": 9612, + "smar": 3201, + "smart": 5383, + "smart": 4115, + "smartcities": 34822, + "smartcity": 33973, + "smarter": 18990, + "smartest": 37092, + "smarthome": 47726, + "smartphone": 11290, + "smartphones": 22212, + "smartwatch": 35798, + "smash": 17258, + "smash": 10332, + "smashbros": 44897, + "smashed": 18410, + "smashes": 45657, + "smashing": 19632, + "smatter": 16537, + "smb": 30446, + "smc": 31375, + "smc": 28312, + "smd": 34582, + "sme": 11758, + "sme": 15650, + "smear": 37546, + "smel": 28476, + "smell": 9688, + "smelling": 32493, + "smells": 14668, + "smelly": 46145, + "smen": 15961, + "smer": 48526, + "smere": 39629, + "smes": 26141, + "smg": 46876, + "smh": 9623, + "smi": 5655, + "smi": 40049, + "smil": 33937, + "smile": 27641, + "smile": 3490, + "smiled": 34362, + "smiles": 8726, + "smiley": 22925, + "smiling": 9200, + "smir": 24667, + "smith": 10527, + "smith": 2915, + "smiths": 27872, + "smithson": 25372, + "smithsonian": 31209, + "smm": 19510, + "smma": 42370, + "smo": 2513, + "smo": 13437, + "smobile": 38923, + "smog": 44425, + "smoke": 20381, + "smoke": 6664, + "smoked": 11161, + "smoker": 32348, + "smokers": 29571, + "smokes": 40336, + "smokey": 23670, + "smokin": 32825, + "smoking": 9038, + "smoky": 25549, + "smol": 29939, + "smol": 40403, + "smoo": 5430, + "smooth": 10958, + "smooth": 8990, + "smoother": 44271, + "smoothie": 16668, + "smoothies": 34458, + "smoothly": 32380, + "smore": 48323, + "smp": 32260, + "smriti": 49227, + "sms": 10409, + "smt": 26672, + "smtown": 26072, + "smu": 10878, + "smu": 30458, + "smug": 41021, + "smugg": 28130, + "smuggling": 34146, + "smur": 24708, + "smusic": 19191, + "smw": 44929, + "smx": 46699, + "smy": 14381, + "smyth": 44822, + "sn": 1672, + "sn": 5844, + "sna": 4032, + "snack": 47548, + "snack": 10039, + "snacking": 46474, + "snacks": 12349, + "snag": 34789, + "snag": 28043, + "snagged": 48534, + "snail": 23132, + "snails": 34928, + "snake": 30133, + "snake": 8798, + "snakes": 19605, + "snap": 4578, + "snap": 7404, + "snapback": 31234, + "snapchat": 7799, + "snapmatic": 45907, + "snapp": 10185, + "snapped": 15543, + "snapper": 31677, + "snapping": 31581, + "snaps": 16890, + "snapshot": 18243, + "snar": 30810, + "snare": 40651, + "snat": 18457, + "snatch": 35302, + "snatched": 44821, + "snation": 14362, + "snazzy": 48963, + "snc": 39918, + "sne": 3791, + "sne": 46503, + "sneak": 27871, + "sneak": 6917, + "sneaker": 31698, + "sneaker": 24781, + "sneakers": 17397, + "sneaking": 34633, + "sneakpeek": 47831, + "sneaks": 40926, + "sneaky": 21293, + "snee": 42095, + "snell": 46410, + "sner": 31424, + "snes": 26667, + "snews": 18623, + "snf": 47651, + "sng": 41549, + "snhl": 43093, + "sni": 7186, + "sni": 35570, + "snickers": 49127, + "sniff": 37841, + "snip": 42954, + "sniper": 22157, + "snippet": 37531, + "snippets": 44001, + "snl": 16011, + "sno": 8567, + "sno": 17802, + "snoo": 11352, + "snooker": 25657, + "snoop": 44503, + "snoop": 27754, + "snoopdogg": 48388, + "snoopy": 41967, + "snooze": 40718, + "snor": 16590, + "snoring": 44560, + "snorkel": 44285, + "snorkeling": 48103, + "snow": 3880, + "snow": 2583, + "snowball": 39254, + "snowboard": 33403, + "snowboarding": 32397, + "snowday": 37982, + "snowden": 32154, + "snowdon": 47107, + "snowdonia": 36088, + "snowed": 45073, + "snowfall": 21714, + "snowflake": 33447, + "snowflakes": 38618, + "snowing": 21443, + "snowman": 22668, + "snowstorm": 38777, + "snowy": 14191, + "snp": 15301, + "sns": 36343, + "snsd": 27961, + "snt": 34834, + "snu": 9694, + "snuck": 36522, + "snug": 45169, + "snuggle": 31327, + "snuggles": 48165, + "sny": 17526, + "snyder": 22106, + "snz": 37678, + "so": 759, + "so": 706, + "soa": 39584, + "soak": 24839, + "soaked": 26592, + "soaking": 26750, + "soap": 26086, + "soap": 11088, + "soaps": 40958, + "soar": 48997, + "soar": 22241, + "soaring": 27968, + "soars": 41348, + "sob": 24900, + "sob": 35507, + "sobbing": 36691, + "sober": 30969, + "sober": 24487, + "sobre": 42768, + "sobri": 49308, + "sobs": 43636, + "soc": 3253, + "soc": 7741, + "soca": 49239, + "socal": 46470, + "socal": 20450, + "soccer": 16268, + "soccer": 4233, + "socceroos": 41997, + "socent": 30831, + "sochi": 21014, + "soci": 1720, + "social": 4803, + "social": 2346, + "socialism": 23372, + "socialist": 18450, + "socialists": 43839, + "socially": 24555, + "socialmedi": 23813, + "socialmedia": 9600, + "socialmediamarketing": 31790, + "societal": 40058, + "societies": 25855, + "society": 3757, + "socio": 44319, + "socio": 42790, + "sociology": 32373, + "sock": 29801, + "sock": 18277, + "socket": 28657, + "socks": 8774, + "socorro": 46409, + "socute": 45086, + "sod": 31435, + "soda": 13533, + "sodium": 29070, + "soe": 44136, + "soe": 25498, + "soever": 34024, + "sof": 1571, + "sof": 41187, + "sofa": 15723, + "soff": 35290, + "soff": 30684, + "sofficial": 20563, + "sofi": 41537, + "sofia": 18914, + "sofinstagram": 17301, + "soft": 12778, + "soft": 3773, + "softball": 8369, + "softer": 44462, + "softhe": 23127, + "softly": 34958, + "software": 35941, + "software": 5847, + "softwitter": 11311, + "sog": 44775, + "soggy": 41168, + "sohn": 49267, + "soho": 47749, + "soho": 17592, + "soi": 40495, + "soil": 33417, + "soil": 9216, + "soils": 34891, + "soir": 43427, + "sok": 43456, + "sol": 1175, + "sol": 9941, + "sola": 40086, + "solace": 42567, + "solar": 16990, + "solar": 5199, + "solareclipse": 44727, + "sold": 33116, + "sold": 3939, + "soldi": 5098, + "soldier": 9355, + "soldiers": 7547, + "sole": 10519, + "sole": 8576, + "soleil": 33148, + "solely": 27913, + "solent": 47783, + "soles": 22682, + "soli": 3911, + "solic": 19369, + "solicitor": 45647, + "solicitors": 46000, + "solid": 30626, + "solid": 6148, + "solidar": 10415, + "solidarity": 10983, + "solidi": 46136, + "solids": 49070, + "solihull": 45293, + "solit": 37039, + "solitaire": 47257, + "solitary": 33094, + "solitude": 33199, + "solo": 17626, + "solo": 5797, + "soloist": 46391, + "solom": 15768, + "solomon": 19785, + "solos": 44868, + "solst": 20298, + "solstice": 21359, + "solu": 2487, + "solution": 4575, + "solutions": 5140, + "solve": 8917, + "solved": 13451, + "solves": 42740, + "solving": 15581, + "som": 734, + "som": 10672, + "soma": 36170, + "somal": 40281, + "somali": 26231, + "somalia": 17051, + "somaliland": 43315, + "some": 1132, + "some": 836, + "somebody": 8305, + "someday": 17127, + "somehow": 11735, + "someone": 2100, + "somer": 9656, + "somerhalder": 33990, + "somerset": 14926, + "somerville": 41409, + "somes": 38124, + "somethin": 33541, + "something": 28316, + "something": 2006, + "sometime": 21464, + "sometimes": 4237, + "somewhat": 17864, + "somewhere": 8119, + "somm": 42726, + "somme": 30625, + "sommer": 44954, + "somos": 24951, + "son": 1176, + "son": 825, + "sona": 21249, + "sonam": 40096, + "sonar": 48235, + "sonata": 37009, + "sone": 29599, + "song": 6868, + "song": 2295, + "songs": 4641, + "songwriter": 13034, + "songwriters": 39583, + "songwriting": 33567, + "songz": 49302, + "soni": 34899, + "soni": 35911, + "sonia": 20409, + "sonic": 23785, + "sonic": 9132, + "sonics": 48511, + "sonja": 46102, + "sonline": 23412, + "sonny": 43000, + "sonny": 20880, + "sono": 44109, + "sonom": 48596, + "sonoma": 26269, + "sons": 5502, + "sonsof": 46676, + "sont": 31063, + "sonthe": 40923, + "sony": 16042, + "sony": 8748, + "sonya": 39172, + "soo": 5517, + "soo": 8602, + "soom": 39771, + "soon": 27559, + "soon": 1745, + "sooner": 18968, + "sooners": 30449, + "sooo": 11526, + "soooo": 13658, + "sooooo": 21199, + "soooooo": 34859, + "soor": 46698, + "soothe": 44424, + "soothing": 27730, + "sop": 3974, + "sop": 19194, + "soph": 34963, + "sophi": 6192, + "sophia": 16790, + "sophie": 38648, + "sophie": 12357, + "sophistic": 17646, + "sophisticated": 20833, + "sophom": 13696, + "sophomore": 15242, + "sophomores": 47645, + "soprano": 28880, + "soproud": 44479, + "sor": 1852, + "sor": 16872, + "sora": 38719, + "sorbet": 39994, + "sore": 43330, + "sore": 15454, + "sored": 6731, + "soren": 38907, + "sorg": 28152, + "sori": 38588, + "sorority": 30059, + "soros": 33248, + "sorren": 44012, + "sorrow": 28020, + "sorrows": 47924, + "sorry": 25745, + "sorry": 3675, + "sorrynotsorry": 37105, + "sort": 8450, + "sorta": 34700, + "sorted": 13221, + "sorting": 19198, + "sorts": 12577, + "sory": 16257, + "sos": 25145, + "sos": 5792, + "sosa": 45433, + "sosfam": 47709, + "sot": 41542, + "sot": 34116, + "sothe": 32145, + "sotho": 45496, + "soto": 27947, + "sotto": 26047, + "sotu": 32286, + "sou": 1101, + "sou": 24293, + "sought": 18874, + "soul": 8701, + "soul": 3755, + "soulful": 30196, + "soulmate": 38130, + "souls": 10951, + "soun": 19474, + "sound": 5236, + "sound": 3608, + "soundcheck": 31394, + "soundcloud": 15190, + "sounded": 28287, + "sounders": 44933, + "sounding": 21351, + "sounds": 5694, + "soundtrack": 11389, + "soup": 7077, + "soups": 45052, + "sour": 2235, + "sour": 12049, + "source": 23698, + "source": 3634, + "sourced": 23340, + "sources": 5124, + "sourcing": 19574, + "sourdough": 29921, + "souri": 11674, + "sous": 32093, + "sousa": 46296, + "sout": 38156, + "sout": 32732, + "south": 2938, + "south": 2045, + "southafrica": 15184, + "southampton": 15767, + "southbank": 44173, + "southbound": 22932, + "southeast": 13942, + "southeastern": 26813, + "southend": 25583, + "souther": 33330, + "southern": 17704, + "southern": 5036, + "southgate": 47262, + "southkorea": 43552, + "southport": 37446, + "southside": 36436, + "southsudan": 30419, + "southwark": 39098, + "southwe": 46443, + "southwest": 13320, + "southwestern": 30157, + "souven": 20210, + "souvenir": 24811, + "souvenirs": 48460, + "souza": 29424, + "sov": 29737, + "sover": 31876, + "sovere": 17736, + "sovereign": 29418, + "sovereign": 26337, + "sovereignty": 31701, + "soviet": 14274, + "sow": 33089, + "sowe": 36130, + "soweto": 47070, + "sown": 49369, + "sox": 39556, + "sox": 8657, + "soy": 16524, + "soy": 15010, + "soybean": 34606, + "soybeans": 40840, + "soyu": 39578, + "soyuz": 43842, + "sp": 588, + "sp": 4393, + "spa": 7852, + "spa": 6692, + "spac": 10336, + "space": 7857, + "space": 2138, + "spacecraft": 25940, + "spaces": 9006, + "spaceship": 34317, + "spacex": 22511, + "spacey": 48770, + "spacious": 24769, + "spad": 45362, + "spade": 32562, + "spades": 48368, + "spaghetti": 18440, + "spain": 5083, + "spal": 26018, + "spam": 29712, + "spam": 14624, + "span": 4270, + "span": 14537, + "spandex": 41686, + "spani": 16721, + "spaniel": 35435, + "spanish": 29966, + "spanish": 6013, + "spann": 25323, + "spanning": 38638, + "spans": 45407, + "spaper": 34548, + "spar": 3378, + "spar": 34576, + "spare": 12615, + "spares": 39505, + "spark": 9555, + "spark": 11047, + "sparked": 32647, + "sparkle": 18287, + "sparkles": 36410, + "sparkling": 17893, + "sparkly": 30542, + "sparks": 15046, + "sparky": 47198, + "sparring": 42161, + "sparrow": 22888, + "spart": 10143, + "sparta": 38401, + "spartan": 26582, + "spartan": 24225, + "spartans": 20457, + "sparty": 36477, + "spas": 31714, + "spati": 19200, + "spatial": 22022, + "spaw": 31605, + "spawn": 29166, + "spay": 40634, + "spc": 20492, + "spca": 37018, + "spd": 37717, + "spd": 28307, + "spdwy": 45981, + "spe": 876, + "spe": 36676, + "speak": 20599, + "speak": 4208, + "speake": 46077, + "speaker": 25764, + "speaker": 4914, + "speakers": 7675, + "speaking": 3714, + "speaks": 5661, + "spear": 23277, + "spear": 30420, + "speare": 43859, + "spears": 20242, + "spec": 1711, + "spec": 18596, + "speci": 1969, + "special": 11422, + "special": 1689, + "specialist": 10630, + "specialists": 21719, + "speciality": 46904, + "specialized": 23265, + "specializes": 48533, + "specially": 4513, + "specials": 11983, + "specialty": 18262, + "species": 6330, + "specific": 10528, + "specifically": 17174, + "specification": 46394, + "specifications": 39705, + "specified": 48114, + "specimen": 30263, + "specimens": 42715, + "specs": 24093, + "spect": 3416, + "spectac": 7242, + "spectacle": 34342, + "spectacular": 8404, + "spectator": 32372, + "spectators": 39306, + "spective": 6633, + "spector": 48676, + "spectral": 45441, + "spectre": 35998, + "spectro": 27646, + "spectrum": 13532, + "specul": 19209, + "speculation": 30898, + "sped": 38813, + "spee": 4050, + "speech": 19556, + "speech": 4902, + "speeches": 25208, + "speechless": 23152, + "speed": 6860, + "speed": 4163, + "speeding": 27264, + "speeds": 22017, + "speedway": 11480, + "speedy": 21603, + "spel": 41887, + "spell": 22784, + "spell": 11230, + "spelled": 24339, + "spelling": 15614, + "spells": 25335, + "spelt": 38316, + "spen": 5087, + "spence": 33324, + "spencer": 27509, + "spencer": 10678, + "spend": 4664, + "spending": 5961, + "spends": 22508, + "spent": 4429, + "speople": 33035, + "sper": 8213, + "sper": 15313, + "sperm": 35781, + "sperson": 22687, + "spf": 34973, + "spg": 34623, + "sph": 28909, + "sph": 24684, + "sphe": 33691, + "spher": 18349, + "sphere": 6987, + "spheres": 37478, + "spheric": 21744, + "sphin": 39237, + "sphinx": 46487, + "spho": 20442, + "sphoto": 38594, + "sphy": 43808, + "spi": 3174, + "spi": 37080, + "spic": 17264, + "spice": 29761, + "spice": 10141, + "spiced": 24267, + "spicer": 37627, + "spices": 21194, + "spicy": 10915, + "spide": 36801, + "spider": 11963, + "spider": 7622, + "spiderman": 39808, + "spiderman": 18427, + "spiders": 23141, + "spidey": 41706, + "spie": 28573, + "spie": 28746, + "spied": 43998, + "spiegel": 45351, + "spiel": 28435, + "spiel": 37690, + "spielberg": 37569, + "spies": 25374, + "spieth": 43254, + "spike": 35306, + "spike": 15310, + "spiked": 47014, + "spikes": 29582, + "spil": 47765, + "spill": 43933, + "spill": 18006, + "spilled": 33206, + "spilling": 49006, + "spills": 35796, + "spin": 6288, + "spin": 9226, + "spinach": 14747, + "spinal": 23925, + "spine": 48221, + "spine": 19646, + "sping": 47113, + "spinner": 29924, + "spinning": 13987, + "spino": 40848, + "spinoff": 42513, + "spinrilla": 46064, + "spins": 27243, + "spion": 39604, + "spionage": 41838, + "spir": 3745, + "spiral": 19873, + "spiration": 38126, + "spire": 27439, + "spired": 40650, + "spires": 46938, + "spiri": 4024, + "spirit": 18224, + "spirit": 4071, + "spirited": 34701, + "spirits": 13192, + "spiritu": 7237, + "spiritual": 46076, + "spiritual": 9473, + "spirituality": 22165, + "spiro": 40085, + "spit": 18115, + "spit": 23177, + "spite": 26060, + "spitfire": 31126, + "spitting": 40721, + "spl": 2470, + "spl": 33052, + "spla": 4809, + "splac": 16059, + "splace": 38743, + "splash": 43641, + "splash": 11879, + "splat": 15733, + "splatoon": 22565, + "splay": 3169, + "splen": 18552, + "splend": 29861, + "splendid": 21016, + "splendor": 46262, + "splin": 38090, + "split": 25443, + "split": 9109, + "splits": 34897, + "splitting": 37210, + "splus": 40866, + "spn": 35467, + "spn": 19414, + "spnfamily": 38566, + "spo": 1261, + "spo": 21085, + "spock": 43918, + "spoil": 25600, + "spoiled": 21399, + "spoiler": 16512, + "spoilers": 18326, + "spoils": 42436, + "spoilt": 35358, + "spokane": 24528, + "spoke": 13890, + "spoke": 6518, + "spoken": 12979, + "spokesman": 31632, + "spokesperson": 26234, + "spol": 22476, + "spol": 8132, + "spoli": 34301, + "spolice": 37406, + "spon": 1715, + "spon": 48216, + "sponge": 22861, + "sponge": 24345, + "spongebob": 25089, + "spons": 5597, + "sponsor": 10424, + "sponsor": 7574, + "sponsored": 7197, + "sponsoring": 16181, + "sponsors": 11005, + "sponsorship": 17632, + "spontaneous": 32465, + "spoo": 11248, + "spooky": 15369, + "spool": 49152, + "spoon": 27001, + "spoon": 14024, + "spoons": 29661, + "spor": 1475, + "spor": 33746, + "sport": 4379, + "sport": 2364, + "sporting": 32620, + "sporting": 8944, + "sports": 6436, + "sports": 2054, + "sportsc": 40114, + "sportscar": 46931, + "sportscenter": 39157, + "sportsman": 39020, + "sportsmanship": 34858, + "sportsnet": 34144, + "sportswear": 39747, + "sporty": 33346, + "spot": 3223, + "spot": 3049, + "spotify": 7193, + "spotlight": 7901, + "spots": 7670, + "spotted": 4533, + "spotter": 30742, + "spotting": 15885, + "spouse": 24724, + "spout": 48993, + "spp": 47567, + "spr": 1536, + "spr": 19417, + "spra": 12966, + "spraw": 46590, + "spray": 37885, + "spray": 10449, + "sprayed": 40022, + "spraying": 39224, + "spre": 18740, + "spread": 20620, + "spread": 5284, + "spreading": 11821, + "spreads": 27579, + "spree": 21851, + "spri": 35498, + "spride": 26685, + "spring": 5166, + "spring": 2420, + "springbreak": 37753, + "springer": 30117, + "springfield": 16599, + "springs": 7308, + "springst": 32132, + "springsteen": 28367, + "springtime": 28285, + "springtraining": 49364, + "springwatch": 29239, + "sprink": 15817, + "sprinkle": 42897, + "sprinkler": 48754, + "sprinkles": 37326, + "sprint": 29248, + "sprint": 10751, + "sprinter": 36947, + "sprints": 36404, + "sprite": 32544, + "spro": 13902, + "spro": 37403, + "sproject": 37802, + "sproud": 37686, + "sprout": 35863, + "sprouts": 25756, + "spru": 17041, + "spruce": 23812, + "sprung": 32968, + "sps": 13869, + "spu": 23566, + "spun": 47922, + "spun": 32852, + "spur": 15206, + "spur": 20361, + "spurs": 10916, + "spursofficial": 45290, + "sput": 47521, + "spx": 20584, + "spy": 13861, + "spy": 6656, + "spyder": 39952, + "spying": 36227, + "sq": 9370, + "sq": 11590, + "sqft": 41912, + "sql": 42759, + "sql": 18938, + "sqm": 47978, + "sqn": 41209, + "squ": 1653, + "squad": 13892, + "squad": 4234, + "squadron": 18579, + "squads": 36590, + "square": 19314, + "square": 3999, + "squared": 32967, + "squares": 26972, + "squash": 13312, + "squat": 44628, + "squat": 30680, + "squats": 40213, + "sque": 9721, + "sque": 8097, + "squee": 14420, + "squeeze": 21684, + "squeezed": 40413, + "squid": 42057, + "squid": 22553, + "squir": 9683, + "squire": 48090, + "squirrel": 14004, + "squirrels": 26623, + "squish": 42607, + "squishy": 47001, + "sr": 3437, + "sr": 5428, + "srbachchan": 32353, + "src": 23445, + "sre": 17748, + "sri": 11051, + "sri": 9276, + "sridevi": 46301, + "srilan": 15559, + "srilanka": 16922, + "srin": 26818, + "srinagar": 33671, + "srini": 41899, + "sriracha": 42743, + "sris": 27851, + "srisri": 32966, + "srk": 44982, + "srk": 11216, + "srl": 33808, + "srp": 43004, + "srs": 41764, + "srsly": 44179, + "srt": 28139, + "sru": 44152, + "srugby": 40526, + "ss": 690, + "ss": 632, + "ssa": 6088, + "ssal": 31330, + "ssal": 35936, + "ssb": 37511, + "ssc": 21692, + "ssc": 20364, + "ssd": 23107, + "sse": 9030, + "sse": 8938, + "ssed": 38755, + "ssed": 1804, + "ssel": 17402, + "ssel": 19373, + "sseldorf": 47792, + "ssell": 42388, + "ssels": 8355, + "ssen": 39408, + "ssen": 22645, + "sser": 20445, + "sses": 1802, + "ssett": 44103, + "ssf": 33239, + "ssg": 40707, + "ssh": 48866, + "ssi": 834, + "ssi": 14953, + "ssia": 22238, + "ssian": 31218, + "ssible": 47099, + "ssic": 27774, + "ssic": 17077, + "ssie": 7572, + "ssier": 26422, + "ssil": 15026, + "ssin": 42660, + "ssing": 2112, + "ssion": 16050, + "ssion": 1627, + "ssional": 13727, + "ssionism": 24787, + "ssionist": 27682, + "ssions": 4137, + "ssive": 2734, + "ssively": 28060, + "ssl": 32195, + "ssler": 30287, + "ssly": 24904, + "ssn": 39116, + "ssnhq": 47998, + "sso": 25900, + "sso": 7914, + "ssoccer": 32546, + "sson": 36124, + "sson": 7271, + "ssor": 35152, + "ssp": 31101, + "ssr": 39880, + "sss": 11176, + "ssss": 30676, + "ssss": 15880, + "sssss": 24298, + "sst": 40396, + "ssu": 35351, + "ssummit": 49301, + "ssus": 31286, + "ssw": 36937, + "ssy": 22519, + "ssy": 8661, + "st": 522, + "st": 545, + "sta": 1363, + "sta": 2745, + "stab": 7726, + "stab": 29974, + "stabbed": 24534, + "stabbing": 25474, + "stabil": 42576, + "stabili": 23903, + "stability": 16716, + "stable": 44427, + "stable": 10492, + "stables": 34218, + "stac": 10175, + "stacey": 41653, + "stacey": 24262, + "stache": 23616, + "stack": 24723, + "stack": 11257, + "stacked": 24990, + "stacking": 39836, + "stacks": 24734, + "stacy": 26628, + "stad": 15832, + "stad": 16485, + "stade": 38198, + "stadi": 26587, + "stadion": 48815, + "stadium": 3390, + "stadiums": 38852, + "stadt": 22713, + "staf": 2367, + "staff": 31188, + "staff": 2813, + "staffer": 38494, + "staffers": 44994, + "staffing": 32932, + "stafford": 25006, + "staffordshire": 29198, + "staffs": 36098, + "stag": 12088, + "stag": 20277, + "stage": 23182, + "stage": 2170, + "staged": 19906, + "stages": 12297, + "staggering": 37315, + "staging": 27026, + "stagram": 19503, + "stags": 45936, + "stain": 3933, + "stain": 14603, + "stained": 13751, + "staining": 32523, + "stainless": 12320, + "stains": 32008, + "stair": 7240, + "stair": 17662, + "staircase": 22777, + "stairs": 9577, + "stairway": 45559, + "stak": 39144, + "stake": 15955, + "stake": 7937, + "stakeholder": 39122, + "stakeholders": 22968, + "stakes": 7519, + "staking": 47082, + "stal": 3861, + "stal": 5535, + "stale": 42471, + "stalert": 25450, + "stalin": 28346, + "stalk": 40826, + "stalk": 14878, + "stalker": 26777, + "stalking": 24721, + "stalks": 45886, + "stall": 24636, + "stall": 12058, + "stalled": 40362, + "stallion": 28273, + "stallions": 44787, + "stallone": 40969, + "stalls": 25427, + "stam": 4663, + "stamatic": 30904, + "stamford": 27843, + "stamina": 48753, + "stamp": 28694, + "stamp": 12771, + "stampcollecting": 42852, + "stamped": 38356, + "stampede": 25384, + "stamps": 13827, + "stan": 2203, + "stan": 2434, + "stana": 33311, + "stanbul": 11231, + "stance": 48900, + "stance": 3542, + "stances": 15054, + "stand": 1819, + "stand": 2087, + "standalone": 44887, + "standard": 35780, + "standard": 5807, + "standardi": 30247, + "standards": 9022, + "standby": 36184, + "standing": 39934, + "standing": 2862, + "standings": 19835, + "standoff": 31821, + "standout": 23131, + "standre": 48309, + "stands": 6446, + "standup": 35108, + "standup": 24964, + "standwith": 19540, + "stanford": 36219, + "stanford": 15087, + "stang": 12536, + "stani": 38228, + "stanis": 37711, + "stanley": 19048, + "stanley": 10079, + "stanleycup": 28662, + "stans": 26564, + "stant": 41576, + "stant": 4906, + "stanton": 25400, + "stap": 10438, + "staple": 22695, + "staples": 23646, + "stapleton": 45228, + "star": 993, + "star": 1565, + "starbuck": 48519, + "starbucks": 9499, + "starch": 47837, + "starcraft": 48871, + "stardom": 44616, + "stardust": 34337, + "stare": 18094, + "stared": 47772, + "stares": 37916, + "starfish": 44283, + "stargate": 41099, + "stargazing": 49328, + "staring": 13800, + "stark": 40446, + "stark": 15353, + "starlight": 32197, + "starling": 46205, + "starmagic": 48023, + "starplus": 37815, + "starr": 19186, + "starred": 24180, + "starrer": 41311, + "starring": 6660, + "starry": 30963, + "stars": 2895, + "starship": 37166, + "start": 17466, + "start": 1572, + "started": 2760, + "starter": 7800, + "starters": 22222, + "starting": 2530, + "startrek": 30642, + "startrek": 15349, + "starts": 3105, + "startu": 6996, + "startup": 18049, + "startup": 5882, + "startups": 9056, + "starve": 46957, + "starving": 30473, + "starwar": 17287, + "starwars": 26239, + "starwars": 7887, + "starz": 25928, + "stas": 19866, + "stash": 27711, + "stasy": 45942, + "stat": 3004, + "stat": 15216, + "state": 3492, + "state": 1295, + "statec": 33931, + "stated": 19629, + "statedept": 41458, + "statefair": 40305, + "statement": 5401, + "statements": 19513, + "staten": 38263, + "stateof": 35195, + "states": 22125, + "states": 4218, + "statesman": 35301, + "stateu": 44248, + "statewide": 29561, + "stati": 9622, + "static": 16363, + "stating": 35147, + "station": 13498, + "station": 2631, + "stationary": 29493, + "stationed": 47618, + "stationery": 33851, + "stations": 10051, + "statistical": 29349, + "statistics": 14165, + "stats": 7294, + "statu": 32481, + "statue": 8222, + "statues": 24363, + "status": 6414, + "stau": 28550, + "staur": 3709, + "stav": 20285, + "stax": 32235, + "stay": 4714, + "stay": 2277, + "stayed": 13805, + "staying": 8993, + "stays": 13311, + "staytuned": 39285, + "stc": 29859, + "std": 30477, + "ste": 795, + "ste": 2686, + "stea": 46614, + "stead": 16101, + "stead": 11031, + "steadily": 35049, + "steady": 12937, + "steak": 26955, + "steak": 8913, + "steakhouse": 35031, + "steaks": 30655, + "steal": 37070, + "steal": 10181, + "stealing": 14242, + "steals": 20224, + "stealth": 25327, + "steam": 10962, + "steam": 6972, + "steamboat": 41121, + "steamed": 29007, + "steamer": 49075, + "steaming": 43746, + "steampunk": 24130, + "steamy": 43104, + "stec": 46713, + "stech": 48949, + "stech": 32455, + "sted": 20426, + "sted": 1356, + "stee": 31793, + "steed": 48293, + "steel": 6938, + "steel": 4726, + "steele": 19460, + "steelers": 14430, + "steen": 42851, + "steen": 18625, + "steep": 28648, + "steep": 20714, + "steer": 27612, + "steering": 19833, + "stef": 29158, + "stefan": 15004, + "stefan": 18829, + "stefani": 38319, + "stefano": 30719, + "steff": 30075, + "stein": 13653, + "stein": 5818, + "steiner": 36314, + "stel": 9102, + "stel": 10798, + "stell": 22355, + "stella": 46178, + "stella": 17869, + "stellar": 13810, + "stellen": 42754, + "stem": 24342, + "stem": 6761, + "stemc": 40486, + "stems": 31503, + "sten": 7652, + "sten": 7877, + "stencil": 47854, + "stennis": 45636, + "step": 15572, + "step": 3348, + "steph": 3522, + "steph": 16251, + "stephan": 37312, + "stephani": 48121, + "stephanie": 14361, + "stephen": 10421, + "stephen": 6078, + "stephenking": 46361, + "stephens": 22256, + "stephenson": 37280, + "stepped": 18384, + "stepping": 15906, + "steps": 5408, + "ster": 1022, + "ster": 881, + "stere": 9229, + "stered": 6935, + "stereo": 15992, + "stereo": 17400, + "stereotypes": 27890, + "steria": 38804, + "stering": 14175, + "sterling": 45790, + "sterling": 9378, + "stern": 36254, + "stern": 2945, + "steroids": 37670, + "sterone": 39418, + "sters": 2132, + "stery": 24232, + "stest": 8556, + "stev": 11640, + "steve": 7412, + "steve": 3803, + "steven": 10973, + "steven": 8016, + "stevens": 13877, + "stevenson": 25091, + "stevie": 42104, + "stevie": 18969, + "stew": 17906, + "stewar": 28453, + "steward": 34980, + "steward": 43355, + "stewards": 49294, + "stewardship": 36720, + "stewart": 8120, + "stfu": 47000, + "stg": 48387, + "stgeorge": 43698, + "sth": 13456, + "sth": 34004, + "sthe": 16491, + "sthel": 42863, + "sti": 860, + "sti": 12439, + "stia": 26492, + "stible": 25835, + "stic": 5868, + "stic": 1561, + "stical": 16660, + "stically": 19041, + "stick": 5483, + "stick": 4987, + "sticker": 11270, + "stickers": 11613, + "sticking": 21021, + "sticks": 10016, + "sticky": 18887, + "stics": 5449, + "stie": 38164, + "stie": 11000, + "stier": 42069, + "sties": 16428, + "stiff": 43471, + "stiff": 21441, + "stig": 4088, + "stig": 42551, + "stigate": 15390, + "stigma": 20619, + "stik": 42247, + "stil": 21790, + "stil": 37519, + "stiles": 33028, + "still": 13209, + "still": 1170, + "stills": 20259, + "stim": 18269, + "stime": 24711, + "stimul": 16434, + "stimulate": 42380, + "stimulating": 41237, + "stimulation": 39530, + "stimulus": 47283, + "stin": 2588, + "stin": 4025, + "stina": 22359, + "stine": 7098, + "sting": 19868, + "sting": 1271, + "stingly": 49332, + "stingray": 43229, + "stink": 38213, + "stinky": 44957, + "stino": 40658, + "stint": 33531, + "stion": 10812, + "stip": 39869, + "stips": 44756, + "stique": 43305, + "stir": 12416, + "stir": 19564, + "stirling": 23128, + "stirring": 39205, + "stis": 45224, + "stit": 14110, + "stitch": 30003, + "stitch": 14771, + "stitched": 36540, + "stitcher": 48204, + "stitches": 32360, + "stitching": 45208, + "stitu": 14585, + "stitutes": 40479, + "stive": 22426, + "stix": 48829, + "stjohn": 36153, + "stl": 14179, + "stl": 12527, + "stlblues": 44138, + "stlcards": 28644, + "stle": 7698, + "stles": 48638, + "stlouis": 40358, + "stlouis": 39516, + "stm": 28333, + "stn": 27175, + "sto": 928, + "sto": 5723, + "stock": 5899, + "stock": 3206, + "stocked": 23552, + "stockholm": 16024, + "stocki": 42944, + "stocking": 17335, + "stockings": 28040, + "stockmarket": 40359, + "stockport": 35569, + "stocks": 9321, + "stockton": 26130, + "stoday": 22392, + "stok": 43782, + "stoke": 31338, + "stoke": 13550, + "stoked": 13160, + "stokes": 27512, + "stol": 11401, + "stol": 6700, + "stole": 10995, + "stolen": 8704, + "stolic": 45020, + "stom": 2343, + "stom": 38068, + "stoma": 43545, + "stomach": 14722, + "stomp": 40165, + "stomping": 46144, + "ston": 4101, + "ston": 1839, + "stone": 7694, + "stone": 2441, + "stoned": 36248, + "stonehenge": 42417, + "stoner": 35131, + "stoner": 29115, + "stones": 42659, + "stones": 6885, + "stonewall": 39688, + "stoney": 44198, + "stony": 41717, + "stony": 35691, + "stoo": 24505, + "stood": 9151, + "stool": 34413, + "stool": 22314, + "stop": 6005, + "stop": 1691, + "stopbrexit": 48680, + "stopp": 15738, + "stopped": 6015, + "stopper": 32147, + "stoppers": 34457, + "stopping": 10735, + "stops": 9822, + "stopthe": 26463, + "stor": 809, + "stor": 17740, + "storage": 6824, + "store": 17769, + "store": 2183, + "stored": 28257, + "stores": 6370, + "storey": 24025, + "storians": 34628, + "stories": 3784, + "storing": 40087, + "stork": 46452, + "storm": 7434, + "storm": 2819, + "stormed": 45939, + "stormhour": 12161, + "storming": 24842, + "storms": 6464, + "stormtrooper": 49218, + "stormy": 20075, + "stors": 7178, + "story": 6512, + "story": 1134, + "storyline": 37079, + "storymonth": 23717, + "storyteller": 35882, + "storytelling": 14457, + "storytime": 44197, + "stos": 19281, + "stou": 37168, + "stour": 37361, + "stour": 21928, + "stout": 16550, + "stove": 21423, + "stow": 44284, + "stow": 17046, + "stowe": 34196, + "stown": 28071, + "stown": 7939, + "stp": 30576, + "stpatrick": 21343, + "stpatricksday": 22747, + "str": 807, + "str": 15913, + "stra": 1894, + "stra": 6253, + "strack": 46861, + "strada": 31134, + "strade": 48968, + "straigh": 31016, + "straight": 22114, + "straight": 4241, + "strain": 16887, + "strains": 38067, + "strait": 22946, + "straits": 41984, + "stral": 23289, + "stralia": 42510, + "stran": 18411, + "strand": 18214, + "strand": 17826, + "stranded": 22975, + "strang": 11138, + "strange": 33380, + "strange": 7288, + "strangely": 37566, + "stranger": 35541, + "stranger": 14149, + "strangers": 20684, + "strangerthings": 43271, + "strangest": 46740, + "strap": 13946, + "strapped": 40922, + "straps": 31213, + "stras": 36814, + "stras": 42125, + "strasbourg": 39576, + "strat": 11345, + "strat": 32925, + "strata": 47278, + "strate": 3532, + "strate": 28758, + "strategi": 49102, + "strategic": 10246, + "strategically": 45706, + "strategies": 9942, + "strategist": 37180, + "strategy": 5637, + "strates": 45724, + "stratford": 23955, + "strath": 21997, + "stration": 3156, + "strato": 28878, + "strauss": 32033, + "strava": 34625, + "stravel": 43494, + "straw": 7430, + "straw": 16438, + "strawberries": 17796, + "strawberry": 10233, + "straws": 33048, + "stray": 30784, + "stray": 15712, + "stre": 1079, + "stre": 19652, + "stread": 27797, + "streak": 11749, + "streaks": 42092, + "stream": 8659, + "stream": 3322, + "streamed": 26280, + "streamer": 25178, + "streamers": 19937, + "streaming": 6278, + "streamline": 44917, + "streams": 13545, + "stree": 35082, + "stree": 32438, + "streep": 38701, + "street": 4839, + "street": 2012, + "streetart": 12948, + "streetcar": 34268, + "streetfood": 44486, + "streetphotography": 20786, + "streets": 6058, + "streetstyle": 39118, + "streetwear": 37298, + "strel": 39685, + "stren": 4349, + "streng": 4472, + "strength": 15475, + "strength": 5959, + "strengthen": 16318, + "strengthened": 47131, + "strengthening": 23475, + "strengthens": 40280, + "strengths": 29268, + "stress": 17297, + "stress": 5843, + "stressed": 16497, + "stresses": 32112, + "stressful": 24268, + "stressing": 35917, + "stret": 12265, + "stretch": 10064, + "stretched": 29393, + "stretches": 32231, + "stretching": 24423, + "stri": 1493, + "stri": 27795, + "stria": 39620, + "strial": 30217, + "strian": 12924, + "stric": 2607, + "strick": 25181, + "strickland": 48939, + "strict": 21585, + "strictly": 16475, + "stride": 36024, + "strides": 37355, + "stries": 18171, + "strife": 46473, + "strike": 20774, + "strike": 5767, + "striker": 12448, + "strikers": 33465, + "strikes": 9280, + "striking": 13392, + "string": 25512, + "string": 9696, + "strings": 15699, + "strip": 9317, + "stripe": 19368, + "striped": 22192, + "stripes": 14239, + "stripped": 26602, + "stripper": 45759, + "stripping": 48588, + "strips": 19000, + "strive": 22140, + "striving": 37671, + "stro": 3121, + "stro": 6186, + "stroke": 44621, + "stroke": 10403, + "strokes": 26595, + "strol": 30123, + "stroll": 15924, + "stroller": 47076, + "strolling": 40911, + "strom": 14707, + "stron": 4165, + "strong": 10436, + "strong": 2389, + "stronger": 27760, + "stronger": 9245, + "strongertogether": 38532, + "strongest": 16171, + "strongh": 38678, + "strongly": 15507, + "strophy": 47912, + "strou": 48425, + "stroud": 39895, + "strous": 23752, + "stru": 1666, + "struc": 3311, + "struck": 10861, + "struction": 12497, + "structural": 16899, + "structure": 5285, + "structured": 27147, + "structures": 14171, + "structuring": 37496, + "strugg": 5176, + "struggle": 8443, + "struggled": 32921, + "struggles": 17446, + "struggling": 12135, + "struly": 34118, + "strum": 37632, + "strung": 46033, + "strust": 23920, + "strut": 48375, + "stry": 17325, + "stry": 2245, + "sts": 1088, + "stu": 858, + "stu": 23531, + "stuart": 32054, + "stuart": 11723, + "stub": 27066, + "stubborn": 38955, + "stuck": 6596, + "stud": 22368, + "stud": 13319, + "studded": 29153, + "studen": 44156, + "student": 14681, + "student": 2556, + "students": 1712, + "studi": 5691, + "studied": 21369, + "studies": 6426, + "studio": 17798, + "studio": 3155, + "studios": 6231, + "studs": 27571, + "study": 21051, + "study": 3123, + "studyabroad": 45425, + "studying": 8826, + "stuff": 46072, + "stuff": 3487, + "stuffed": 11781, + "stuffing": 31612, + "stuffs": 43455, + "stuk": 32424, + "stumb": 16784, + "stumble": 39045, + "stumbled": 21776, + "stump": 32064, + "stun": 3088, + "stun": 37959, + "stunned": 34034, + "stunner": 29965, + "stunning": 3769, + "stunningly": 47515, + "stuns": 43796, + "stunt": 19905, + "stunts": 40118, + "stupi": 18975, + "stupid": 42600, + "stupid": 8085, + "stupidity": 33766, + "stur": 10676, + "sturdy": 43780, + "stures": 27223, + "sturgeon": 31580, + "sturi": 21747, + "sturridge": 45331, + "stutt": 30444, + "stuttgart": 32219, + "stv": 27060, + "stv": 9708, + "stweet": 46832, + "stweets": 39174, + "stx": 42548, + "sty": 1421, + "sty": 2920, + "style": 12356, + "style": 1844, + "styled": 17974, + "styles": 6948, + "styli": 38577, + "styling": 14597, + "stylish": 10378, + "stylist": 15928, + "styn": 41394, + "su": 605, + "su": 2937, + "sua": 42448, + "suarez": 21437, + "suave": 47305, + "sub": 1783, + "sub": 7765, + "subaru": 21319, + "subjec": 16090, + "subject": 10300, + "subjects": 22099, + "subli": 16350, + "sublime": 22367, + "submarine": 19968, + "submer": 27156, + "submerged": 43171, + "submission": 16571, + "submissions": 21566, + "submit": 10423, + "submitted": 15189, + "submitting": 38788, + "subram": 49207, + "subs": 16398, + "subscri": 5838, + "subscribe": 9839, + "subscribed": 44867, + "subscriber": 36292, + "subscribers": 17337, + "subscription": 17979, + "subscriptions": 47162, + "subsequ": 33598, + "subsequent": 44323, + "subsi": 14856, + "subsidi": 45029, + "subsidiary": 45506, + "subsidies": 37685, + "subsidy": 47462, + "substan": 17487, + "substance": 19309, + "substances": 36834, + "substantial": 27171, + "substantially": 47577, + "substitu": 18529, + "substitute": 25340, + "subtitles": 39479, + "subtle": 16536, + "subur": 12517, + "suburb": 37664, + "suburban": 23570, + "suburbs": 25317, + "subway": 12196, + "suc": 1869, + "succe": 7981, + "succeed": 13556, + "succeeded": 41077, + "succes": 39019, + "success": 3695, + "success": 3034, + "successes": 29436, + "successful": 4670, + "successfully": 9934, + "succession": 38491, + "successive": 41319, + "successor": 34774, + "succu": 45253, + "succul": 25671, + "succulent": 35236, + "such": 2046, + "suction": 42786, + "sud": 8067, + "sud": 33714, + "sudan": 31149, + "sudan": 13474, + "sudanese": 42837, + "sudbury": 32488, + "sudden": 10833, + "sudden": 15433, + "suddenly": 11076, + "sue": 14045, + "sue": 6641, + "sued": 22225, + "suede": 21036, + "sues": 17105, + "suf": 21204, + "suf": 22579, + "sufc": 37091, + "suff": 4866, + "suffe": 13510, + "suffer": 13557, + "suffered": 14766, + "suffering": 10140, + "suffers": 22389, + "sufficient": 28410, + "suffol": 13775, + "suffolk": 46408, + "suffolk": 15685, + "suffra": 34596, + "suffrage": 39567, + "sufi": 39756, + "sug": 3189, + "suga": 28757, + "sugar": 12418, + "sugar": 5574, + "sugge": 6345, + "suggest": 13356, + "suggested": 18790, + "suggesti": 15033, + "suggesting": 29792, + "suggestion": 23741, + "suggestions": 16052, + "suggests": 13333, + "suho": 32744, + "sui": 24972, + "suici": 16372, + "suicidal": 37165, + "suicide": 31310, + "suicide": 8247, + "suing": 18309, + "suisse": 35964, + "suit": 11887, + "suit": 3940, + "suitable": 17476, + "suitcase": 27792, + "suite": 9346, + "suited": 25919, + "suites": 21523, + "suits": 9949, + "suk": 24820, + "suk": 6886, + "suka": 44017, + "suke": 25590, + "sukh": 46961, + "suki": 32704, + "sul": 1767, + "sul": 19879, + "sula": 34713, + "sula": 26143, + "sullivan": 14477, + "sully": 37752, + "sulph": 37234, + "sulphur": 47659, + "sultan": 35650, + "sultan": 17049, + "sum": 7054, + "sum": 8257, + "suma": 47938, + "sumat": 32640, + "sumatra": 47346, + "sume": 45457, + "sumi": 41248, + "summ": 1309, + "summar": 34657, + "summari": 31993, + "summary": 13435, + "summed": 34912, + "summer": 5500, + "summer": 1673, + "summers": 18254, + "summerslam": 40264, + "summertime": 19025, + "summit": 30011, + "summit": 3768, + "summon": 27622, + "summon": 39782, + "sumner": 46813, + "sumo": 33734, + "sump": 34252, + "sumptuous": 47354, + "sums": 13325, + "sun": 968, + "sun": 2176, + "sunbathing": 46994, + "sunburn": 45767, + "sund": 40735, + "sundae": 38078, + "sundance": 24128, + "sundar": 44936, + "sunday": 6649, + "sunday": 1706, + "sundayfunday": 21565, + "sundaymorning": 24809, + "sundaymotivation": 46227, + "sundays": 15827, + "sundaywith": 26469, + "sundaywithmarsha": 26662, + "sunder": 15097, + "sunderland": 45727, + "sunderland": 18851, + "sundown": 44438, + "sune": 41096, + "sunflower": 21559, + "sunflowers": 39809, + "sung": 16903, + "sung": 6047, + "sunglasses": 12906, + "suni": 17663, + "suni": 47010, + "sunil": 32861, + "sunite": 21382, + "sunited": 35276, + "sunk": 37534, + "sunken": 43473, + "sunlight": 17996, + "sunni": 44315, + "sunny": 15632, + "sunny": 5438, + "sunrise": 5610, + "suns": 18322, + "sunscreen": 29355, + "sunset": 37880, + "sunset": 3424, + "sunsets": 17721, + "sunshine": 32761, + "sunshine": 5385, + "suny": 41308, + "sup": 19078, + "sup": 8249, + "supdates": 24177, + "super": 1642, + "super": 1994, + "superb": 8930, + "superbike": 45709, + "superbowl": 47461, + "superbowl": 16467, + "supercar": 27021, + "supercars": 32185, + "supercell": 43227, + "supercharged": 47479, + "supere": 46831, + "superfood": 41715, + "supergirl": 25771, + "superhero": 14049, + "superheroes": 23334, + "superint": 17615, + "superintendent": 19020, + "superior": 13205, + "superjunior": 40475, + "superleague": 45539, + "superman": 11237, + "supermarket": 19897, + "supermarkets": 45106, + "supermodel": 41963, + "supermoon": 36571, + "supernatural": 15484, + "supernova": 39843, + "superrugby": 48717, + "supersonic": 42019, + "supersport": 46319, + "superst": 38202, + "superstar": 32551, + "superstar": 10472, + "superstars": 25797, + "supervis": 12709, + "supervised": 41316, + "supervision": 36234, + "supervisor": 20366, + "supervisors": 37958, + "superyacht": 42714, + "supp": 1023, + "supper": 15727, + "supple": 31431, + "supplement": 19924, + "supplements": 21265, + "supplied": 24106, + "supplier": 18043, + "suppliers": 24196, + "supplies": 9384, + "supply": 25074, + "supply": 6389, + "supplychain": 31224, + "supplying": 32739, + "suppo": 6941, + "suppor": 2104, + "support": 12062, + "support": 1425, + "supported": 8038, + "supporter": 12992, + "supporters": 7403, + "supportindiefilm": 43976, + "supporting": 3976, + "supportive": 18313, + "supportlocal": 43852, + "supports": 8336, + "supportsmall": 30941, + "supportsmallstreamers": 36097, + "suppose": 18924, + "supposed": 9119, + "supposedly": 32302, + "suppre": 20542, + "suppression": 36508, + "supra": 48485, + "supre": 5875, + "supremac": 28643, + "supremacist": 39005, + "supremacy": 28913, + "supreme": 35222, + "supreme": 7468, + "supt": 23625, + "sur": 1090, + "sur": 7123, + "sura": 33412, + "sura": 49125, + "surabaya": 45227, + "surance": 22184, + "surat": 30201, + "sure": 14320, + "sure": 1650, + "sured": 36869, + "surely": 11409, + "sures": 12725, + "suresh": 32118, + "suresh": 31464, + "sureshpp": 41924, + "sureshpprabhu": 42050, + "surf": 10176, + "surf": 10322, + "surface": 7744, + "surfaces": 20746, + "surfer": 24925, + "surfers": 34842, + "surfing": 15762, + "surg": 13045, + "surge": 17457, + "surgeon": 16039, + "surgeons": 26000, + "surger": 5122, + "surgeries": 34940, + "surgery": 5344, + "surgical": 16386, + "suri": 14130, + "suri": 33952, + "suring": 16817, + "suriya": 17832, + "surpass": 45494, + "surpassed": 25648, + "surplus": 29413, + "surpri": 3244, + "surprise": 5099, + "surprised": 8949, + "surprises": 16920, + "surprising": 14964, + "surprisingly": 17367, + "surreal": 18408, + "surrealism": 41773, + "surrender": 20964, + "surrendered": 44601, + "surrey": 26489, + "surrey": 14315, + "surro": 47499, + "surroun": 8250, + "surround": 26543, + "surround": 22999, + "surrounded": 13589, + "surrounding": 12544, + "surroundings": 26915, + "surrounds": 39012, + "suru": 49240, + "surve": 8952, + "surveill": 15408, + "surveillance": 15578, + "survey": 45914, + "survey": 6809, + "surveying": 33085, + "surveys": 25096, + "survi": 3440, + "surviv": 12922, + "survival": 10172, + "survive": 10431, + "survived": 13483, + "survives": 30927, + "surviving": 18609, + "survivor": 31934, + "survivor": 10944, + "survivors": 13711, + "surya": 37767, + "sus": 8091, + "sus": 3036, + "susa": 20546, + "susan": 19922, + "susan": 10168, + "suscep": 44270, + "sush": 22298, + "sushi": 11729, + "sushmaswar": 48200, + "susie": 32284, + "susp": 7971, + "suspec": 10298, + "suspect": 9065, + "suspected": 15579, + "suspects": 18265, + "suspen": 10578, + "suspend": 41007, + "suspended": 13126, + "suspends": 39535, + "suspense": 21556, + "suspension": 15417, + "suspici": 25714, + "suspicion": 34910, + "suspicious": 19862, + "sussex": 31244, + "sussex": 13266, + "sustain": 4644, + "sustain": 28156, + "sustainability": 9635, + "sustainable": 23645, + "sustainable": 7078, + "sustained": 22699, + "sustaining": 44418, + "sut": 23984, + "sut": 28956, + "sutherland": 27592, + "sutton": 39359, + "sutton": 18564, + "suv": 15985, + "suz": 9957, + "suzanne": 24617, + "suzu": 36289, + "suzuki": 16892, + "suzy": 26552, + "sv": 6508, + "sv": 17083, + "svc": 45065, + "sve": 47637, + "sven": 37786, + "sven": 45183, + "sver": 45923, + "sville": 44580, + "sville": 6741, + "svp": 28465, + "svt": 42014, + "svu": 32123, + "sw": 1220, + "sw": 4457, + "swa": 4707, + "swa": 31916, + "swach": 20862, + "swachhb": 31898, + "swachhbharat": 36927, + "swag": 8852, + "swag": 8177, + "swagg": 47702, + "swagger": 35797, + "swain": 43226, + "swal": 13433, + "swallow": 28979, + "swallowed": 46956, + "swallows": 45124, + "swam": 42539, + "swami": 25021, + "swamp": 41953, + "swamp": 16595, + "swamy": 28445, + "swan": 8215, + "swan": 12530, + "swana": 24699, + "swans": 19516, + "swansea": 16567, + "swanson": 34797, + "swap": 15234, + "swapped": 39077, + "swapping": 44702, + "swaps": 49242, + "swar": 11680, + "swarm": 31577, + "swarovski": 28515, + "swat": 32547, + "swat": 26482, + "swatch": 48053, + "sway": 26443, + "sway": 26617, + "swc": 42231, + "swe": 2350, + "swe": 38070, + "swear": 7406, + "swearing": 32627, + "sweat": 10282, + "sweat": 12663, + "sweater": 11455, + "sweaters": 31303, + "sweating": 33215, + "sweats": 39321, + "sweatshirt": 22442, + "sweaty": 28419, + "sweden": 8760, + "swedish": 11585, + "swee": 1812, + "sweek": 30017, + "sweeney": 27286, + "sweep": 23220, + "sweep": 13669, + "sweeping": 25719, + "sweeps": 26887, + "sweepstakes": 25992, + "sweet": 10957, + "sweet": 2418, + "sweetened": 45577, + "sweeter": 32873, + "sweetest": 15180, + "sweethe": 16316, + "sweetheart": 18079, + "sweetie": 24450, + "sweetness": 29713, + "sweets": 18045, + "swel": 48470, + "swell": 35538, + "swell": 21490, + "swelling": 46578, + "swept": 23311, + "swer": 30514, + "swfc": 30227, + "swfl": 46607, + "swi": 3881, + "swi": 45223, + "swick": 17159, + "swif": 28548, + "swift": 34843, + "swift": 8229, + "swild": 33909, + "swild": 38696, + "swildlife": 46818, + "swim": 4928, + "swim": 7681, + "swimmer": 25475, + "swimmers": 27776, + "swimming": 7411, + "swims": 46798, + "swimsuit": 25504, + "swimwear": 31889, + "swin": 14554, + "swin": 40798, + "swindon": 29540, + "swine": 31166, + "swing": 25292, + "swing": 7429, + "swinging": 26760, + "swings": 29141, + "swipe": 31828, + "swire": 42753, + "swirl": 35795, + "swis": 23611, + "swish": 38571, + "swiss": 37917, + "swiss": 9287, + "swit": 3726, + "switch": 22480, + "switch": 5893, + "switched": 22869, + "switches": 33569, + "switching": 21155, + "swith": 17299, + "switzer": 9835, + "switzerland": 9912, + "swivel": 48256, + "swo": 38673, + "swol": 29575, + "swollen": 36129, + "swoo": 29744, + "swood": 24158, + "swoon": 37028, + "swoop": 45661, + "sword": 33294, + "sword": 11356, + "swords": 27181, + "swork": 42722, + "sworld": 33305, + "sworn": 21130, + "sworth": 13322, + "swt": 38878, + "swx": 20597, + "sx": 9402, + "sx": 17806, + "sxsw": 13369, + "sy": 974, + "sy": 2126, + "sya": 35017, + "sycam": 34911, + "sycamore": 43086, + "syd": 4525, + "syd": 22504, + "sydney": 15878, + "sydney": 5278, + "syed": 27624, + "syfy": 32047, + "sykes": 27287, + "syl": 6452, + "sylla": 41708, + "sylvania": 12011, + "sylve": 28369, + "sylvester": 37214, + "sylvia": 25670, + "sym": 3645, + "sym": 40327, + "symb": 22987, + "symbol": 13085, + "symboli": 22019, + "symbolic": 33177, + "symbolism": 44679, + "symbols": 25476, + "symmetry": 31427, + "symp": 11468, + "sympathi": 47493, + "sympathy": 32477, + "symph": 9544, + "symphonic": 42639, + "symphony": 11180, + "sympo": 9730, + "symposium": 9971, + "symptom": 47799, + "symptoms": 12956, + "syn": 3758, + "syn": 36090, + "synago": 30945, + "synagogue": 33518, + "sync": 20081, + "synchron": 23943, + "syndic": 21098, + "syndicate": 28779, + "syndrome": 10927, + "syner": 22283, + "synergy": 32012, + "syno": 31533, + "synod": 47712, + "synopsis": 47018, + "synth": 33841, + "synth": 24462, + "synthe": 22604, + "synthesi": 33565, + "synthesis": 21602, + "synthesizer": 44077, + "synthetic": 19917, + "syou": 26742, + "syour": 21718, + "syrac": 17279, + "syracuse": 19640, + "syrah": 45364, + "syri": 18917, + "syria": 5563, + "syrian": 47562, + "syrian": 10041, + "syrians": 41392, + "syrup": 16611, + "sys": 26726, + "syste": 1933, + "system": 47813, + "system": 2422, + "systematic": 28586, + "systemic": 33807, + "systems": 4828, + "sz": 13438, + "sz": 15879, + "sze": 44507, + "szn": 48092, + "são": 45911, + "sé": 37879, + "t": 83, + "t": 339, + "ta": 648, + "ta": 1397, + "taa": 43874, + "tab": 2648, + "tab": 14724, + "tabby": 36145, + "tabern": 48991, + "tability": 15770, + "table": 12108, + "table": 2175, + "tableau": 39723, + "tables": 7822, + "tablet": 12494, + "tabletop": 46843, + "tabletop": 25773, + "tablets": 20436, + "tably": 24440, + "taboo": 38400, + "tabs": 29163, + "tac": 3145, + "tac": 22653, + "tache": 39239, + "tack": 6339, + "tack": 34446, + "tackle": 10294, + "tackled": 47218, + "tackles": 18021, + "tackling": 19628, + "taco": 31924, + "taco": 12436, + "tacoma": 25397, + "tacos": 14090, + "tactic": 40377, + "tactical": 17137, + "tactics": 16410, + "tacular": 48985, + "tad": 15890, + "tad": 19860, + "tado": 40846, + "tae": 15257, + "tae": 15580, + "taehyung": 24642, + "taek": 30753, + "taekwondo": 39963, + "taemin": 30600, + "taeyang": 45802, + "taeyeon": 27389, + "taf": 29660, + "taft": 42141, + "tag": 3456, + "tag": 3640, + "tage": 2669, + "tages": 39902, + "tagged": 12969, + "tagging": 25138, + "tagne": 47467, + "tags": 11606, + "tah": 14822, + "tah": 7090, + "tahit": 45385, + "tahoe": 26140, + "tai": 6511, + "tai": 13040, + "taiji": 30185, + "tail": 7156, + "tail": 4132, + "tailed": 20626, + "tailgate": 23168, + "tailgating": 42625, + "tailo": 27230, + "tailor": 29870, + "tailored": 28275, + "tailoring": 46357, + "tails": 16066, + "tain": 2841, + "tain": 1908, + "taine": 21214, + "taine": 32299, + "tained": 10212, + "taining": 7565, + "tainment": 30063, + "tains": 3952, + "tainted": 47211, + "taipei": 24356, + "tair": 29143, + "tairp": 43707, + "tait": 45325, + "taiwan": 36319, + "taiwan": 12626, + "taiwanese": 41416, + "taj": 28937, + "taj": 24805, + "taji": 46358, + "tak": 15070, + "tak": 14458, + "taka": 24070, + "taka": 40968, + "take": 5052, + "take": 1172, + "takeaway": 25737, + "takeaways": 32080, + "takeme": 41748, + "taken": 2807, + "takeoff": 32789, + "takeover": 11863, + "taker": 17939, + "takers": 30775, + "takes": 2633, + "takin": 30890, + "taking": 2019, + "taku": 48168, + "tal": 976, + "tal": 2066, + "tala": 29845, + "talaga": 35349, + "talbot": 30585, + "tale": 33971, + "tale": 7798, + "talent": 30435, + "talent": 5114, + "talented": 5331, + "talents": 16136, + "tales": 9469, + "tali": 12122, + "tali": 45406, + "taliban": 20788, + "talis": 36480, + "tality": 15631, + "talk": 12462, + "talk": 1841, + "talked": 10153, + "talkin": 26040, + "talking": 31463, + "talking": 2578, + "talks": 3237, + "tall": 11664, + "tall": 7771, + "talla": 21528, + "tallade": 44220, + "tallahassee": 37832, + "taller": 23470, + "tallest": 19774, + "tallinn": 45079, + "tally": 16323, + "talon": 47897, + "tam": 2661, + "tam": 12246, + "tama": 45424, + "tamanna": 48055, + "tamar": 22901, + "tamara": 35697, + "tame": 38557, + "tame": 32778, + "tamed": 40575, + "tami": 39429, + "tamil": 23046, + "tamil": 14033, + "tamilnadu": 32371, + "tamine": 42566, + "tammy": 28396, + "tampa": 10906, + "tampab": 37852, + "tamu": 34105, + "tan": 2123, + "tan": 5039, + "tana": 21396, + "tand": 20244, + "tandem": 33756, + "tane": 13344, + "tane": 24923, + "taneous": 22275, + "taneously": 24422, + "tang": 10425, + "tang": 20794, + "tanger": 31844, + "tangerine": 42045, + "tangible": 44823, + "tangle": 36568, + "tangled": 33587, + "tango": 24089, + "tani": 31374, + "tani": 32985, + "tania": 45369, + "tank": 29858, + "tank": 6172, + "tanker": 25020, + "tanks": 14223, + "tann": 19174, + "tanner": 22001, + "tanning": 27985, + "tans": 27332, + "tant": 41383, + "tant": 41695, + "tante": 48262, + "tanto": 45685, + "tany": 34410, + "tanya": 26800, + "tanz": 47399, + "tanzania": 15711, + "tao": 29084, + "tao": 18923, + "tap": 17923, + "tap": 7888, + "tapas": 27361, + "tape": 18332, + "tape": 5749, + "taped": 33219, + "tapes": 17903, + "tapestry": 33525, + "taping": 24355, + "tapp": 27644, + "tapp": 27764, + "tapped": 26649, + "tapping": 27882, + "tapro": 34415, + "taproom": 40266, + "taps": 23267, + "tar": 2002, + "tar": 6977, + "tara": 15264, + "tarak": 37813, + "taran": 32370, + "tarantino": 41180, + "tarde": 48670, + "tardis": 35410, + "tares": 34587, + "targe": 9620, + "target": 38556, + "target": 5400, + "targeted": 14968, + "targeting": 15818, + "targets": 12468, + "tari": 4238, + "tari": 38012, + "tarian": 11762, + "tarians": 42789, + "taries": 47291, + "tariff": 40220, + "tariffs": 28335, + "tariq": 42526, + "tarmac": 44294, + "taro": 26264, + "tarot": 23702, + "tart": 16707, + "tart": 14120, + "tartan": 35064, + "tarts": 29799, + "tary": 31729, + "tary": 5065, + "tarzan": 45463, + "tas": 6538, + "tas": 10163, + "tash": 35272, + "tasha": 44967, + "task": 39189, + "task": 10549, + "tasks": 19453, + "tasmania": 22429, + "tasmanian": 45102, + "tassel": 49276, + "tast": 10839, + "taste": 14314, + "taste": 5219, + "tasted": 22827, + "tasteof": 38097, + "taster": 29743, + "tastes": 13736, + "tastic": 21337, + "tasting": 7656, + "tastings": 49273, + "tasty": 43390, + "tasty": 8568, + "tat": 2652, + "tat": 21592, + "tata": 19300, + "tate": 44476, + "tate": 13295, + "tath": 27566, + "tati": 31433, + "tatiana": 48837, + "tation": 5280, + "tations": 32324, + "tator": 18791, + "tators": 37206, + "tats": 44557, + "tatt": 9232, + "tatted": 41605, + "tattoo": 15980, + "tattoo": 6325, + "tattooed": 28541, + "tattoos": 14900, + "tatum": 26103, + "tau": 6620, + "tau": 20510, + "taught": 9306, + "taun": 23910, + "taunton": 40681, + "taurus": 32881, + "taver": 37776, + "tavern": 18644, + "taw": 33868, + "taw": 40289, + "tawa": 29035, + "tawards": 14351, + "tax": 4581, + "tax": 3879, + "taxation": 36847, + "taxes": 11462, + "taxi": 25160, + "taxi": 11380, + "taxider": 47420, + "taxis": 34009, + "taxpay": 17986, + "taxpayer": 30978, + "taxpayers": 25503, + "tay": 6542, + "tay": 15073, + "taya": 38484, + "tayl": 3913, + "taylor": 9044, + "taylor": 3961, + "taylorswift": 18936, + "tayo": 33941, + "taz": 41475, + "taz": 31870, + "tb": 1990, + "tb": 7490, + "tba": 34363, + "tball": 8390, + "tball": 1467, + "tbc": 31807, + "tbd": 45548, + "tbh": 13238, + "tbi": 45868, + "tbl": 42962, + "tbli": 43664, + "tblightning": 44178, + "tbo": 34255, + "tbr": 46643, + "tbs": 37368, + "tbt": 2950, + "tc": 6820, + "tc": 5454, + "tca": 35116, + "tch": 10744, + "tch": 4048, + "tches": 42001, + "tcm": 21501, + "tcm": 26588, + "tcmparty": 24338, + "tcot": 8995, + "tcs": 39107, + "tcu": 26791, + "td": 20578, + "td": 3192, + "tdf": 21844, + "tdi": 45621, + "tdp": 47009, + "tds": 20238, + "tdsb": 29836, + "te": 600, + "te": 756, + "tea": 41053, + "tea": 3274, + "teach": 2043, + "teach": 6865, + "teacher": 18051, + "teacher": 4008, + "teachers": 5069, + "teaches": 17110, + "teaching": 5141, + "teachings": 32119, + "teal": 22821, + "team": 2085, + "team": 1027, + "teamcanada": 46636, + "teamed": 20590, + "teamgb": 40971, + "teaming": 24392, + "teammate": 17900, + "teammates": 13921, + "teams": 3891, + "teamsisd": 34703, + "teamusa": 28625, + "teamwork": 14657, + "teaparty": 33065, + "teapo": 35745, + "teapot": 40749, + "tear": 15802, + "tear": 11862, + "tearful": 46873, + "tearing": 24785, + "tears": 7688, + "teas": 23003, + "teas": 29314, + "tease": 25163, + "teased": 49122, + "teaser": 8982, + "teasers": 48990, + "teases": 28509, + "teasing": 36507, + "teat": 26376, + "teatime": 48948, + "teatro": 35756, + "teau": 24931, + "tebow": 37797, + "tec": 17381, + "tec": 11612, + "tech": 1782, + "tech": 2061, + "techcrunch": 42110, + "techn": 6252, + "technews": 31787, + "technic": 16639, + "technic": 37666, + "technical": 49231, + "technical": 7582, + "technically": 23180, + "technician": 22540, + "technicians": 35513, + "techno": 2599, + "techno": 17564, + "technological": 23068, + "technologies": 10040, + "technology": 3089, + "techs": 41353, + "ted": 4841, + "ted": 775, + "tedcruz": 27517, + "teddy": 25758, + "teddy": 11798, + "tedly": 8539, + "tedu": 42517, + "tedx": 17950, + "tedx": 41504, + "tee": 12676, + "tee": 3385, + "teed": 13692, + "teen": 5398, + "teen": 4697, + "teenage": 14069, + "teenager": 19338, + "teenagers": 25989, + "teenchoice": 28203, + "teens": 12375, + "teenth": 20249, + "teenwolf": 40067, + "teeny": 41622, + "teer": 48648, + "tees": 9641, + "teessi": 43295, + "teeth": 8225, + "tega": 29508, + "tegr": 39801, + "teh": 18720, + "teh": 29601, + "tehran": 26399, + "tein": 33223, + "tej": 46724, + "tek": 17489, + "tek": 18294, + "tekken": 29843, + "tel": 4978, + "tel": 2226, + "telang": 23469, + "telangana": 26386, + "tele": 3103, + "tele": 32851, + "telecom": 21057, + "telecommunications": 39900, + "telegram": 26780, + "telegraph": 14713, + "telephone": 17243, + "telescope": 19037, + "telethon": 49266, + "televised": 39470, + "television": 8608, + "telford": 38323, + "tell": 16069, + "tell": 2330, + "teller": 20415, + "tellers": 42707, + "telling": 5507, + "tells": 5217, + "tellu": 42511, + "telly": 31475, + "tels": 43607, + "telugu": 22927, + "tely": 5630, + "tem": 2404, + "tem": 17536, + "tema": 45881, + "teme": 43378, + "temp": 2684, + "temp": 11097, + "tempe": 36723, + "temper": 5981, + "temper": 35521, + "temperature": 9543, + "temperatures": 11575, + "tempered": 40521, + "tempest": 36053, + "templ": 16679, + "template": 18591, + "templates": 30498, + "temple": 21841, + "temple": 5620, + "temples": 24024, + "tempo": 19625, + "tempor": 4858, + "temporal": 43656, + "temporarily": 23189, + "temporary": 6513, + "temps": 11668, + "tempt": 28460, + "temptation": 30118, + "tempted": 26226, + "tempting": 34876, + "ten": 1149, + "ten": 2581, + "tenant": 16954, + "tenants": 26023, + "tenay": 45384, + "tenberg": 31329, + "tend": 17630, + "tend": 21252, + "tendency": 47277, + "tender": 23020, + "tender": 9838, + "tenderloin": 42750, + "tenders": 44741, + "tending": 35084, + "tendon": 48459, + "tends": 39962, + "tene": 24868, + "tened": 13682, + "tener": 29054, + "teneri": 28000, + "tenerife": 29401, + "teners": 41307, + "teness": 18018, + "teng": 34016, + "teng": 28474, + "tennant": 29310, + "tennes": 9514, + "tennessee": 10053, + "tennis": 31504, + "tennis": 5298, + "tenor": 30521, + "tens": 14062, + "tense": 23518, + "tension": 15221, + "tensions": 24224, + "tenstein": 49139, + "tent": 18505, + "tent": 10782, + "tentative": 48238, + "tenth": 27483, + "tention": 12191, + "tents": 30730, + "tenure": 30739, + "teo": 18665, + "tep": 31806, + "tequ": 17502, + "tequila": 18510, + "ter": 704, + "ter": 652, + "tera": 15155, + "teras": 44830, + "tere": 11329, + "tered": 49272, + "tered": 4389, + "terence": 33806, + "teresa": 19081, + "teri": 30917, + "teria": 22685, + "terie": 42276, + "tering": 7929, + "term": 40991, + "term": 4780, + "termin": 4766, + "terminal": 11816, + "terminals": 44091, + "terminator": 29609, + "terminology": 48896, + "terms": 8663, + "tern": 41572, + "tern": 12959, + "terns": 25251, + "tero": 20727, + "tero": 24697, + "terps": 41471, + "terr": 3921, + "terra": 22366, + "terra": 18816, + "terrac": 28549, + "terrace": 13820, + "terraces": 47508, + "terracotta": 45123, + "terrain": 20184, + "terran": 43726, + "terre": 33888, + "terre": 27537, + "terrell": 39494, + "terrence": 38746, + "terrestrial": 46299, + "terri": 4504, + "terri": 36722, + "terrible": 9741, + "terribly": 34558, + "terrier": 14455, + "terriers": 47047, + "terrific": 13837, + "terrified": 28204, + "terrifying": 18526, + "territ": 10720, + "territorial": 39163, + "territories": 32846, + "territory": 13936, + "terror": 9596, + "terror": 9327, + "terrori": 6836, + "terrorism": 10583, + "terrorist": 10575, + "terrorists": 12835, + "terry": 19378, + "terry": 8561, + "ters": 24102, + "ters": 1737, + "terti": 48386, + "tery": 4184, + "tes": 8019, + "tes": 3609, + "tesco": 15434, + "tese": 33320, + "tesla": 12254, + "tess": 21807, + "tess": 20840, + "tessa": 32063, + "test": 7738, + "test": 1628, + "testam": 23477, + "testament": 24609, + "tested": 10576, + "tester": 32707, + "testi": 18373, + "testic": 42364, + "testify": 33088, + "testifying": 46347, + "testim": 12553, + "testimonial": 28834, + "testimony": 18672, + "testing": 4967, + "testo": 42428, + "testosterone": 45168, + "tests": 8715, + "tet": 40468, + "tet": 13275, + "tetra": 40902, + "tetris": 45934, + "teu": 47152, + "teuk": 39979, + "teur": 27120, + "tex": 2056, + "tex": 11728, + "texan": 35287, + "texan": 38386, + "texans": 17580, + "texanscheer": 43717, + "texas": 15713, + "texas": 3403, + "texaste": 46469, + "text": 18169, + "text": 4160, + "textbook": 25952, + "textbooks": 44041, + "texted": 29004, + "textile": 19789, + "textiles": 24326, + "texting": 18600, + "texts": 12767, + "texture": 16505, + "textured": 32168, + "textures": 28063, + "tey": 32395, + "tez": 22664, + "tf": 18828, + "tf": 5001, + "tfc": 30186, + "tfl": 29918, + "tford": 22493, + "tful": 17108, + "tfw": 16741, + "tg": 7665, + "tg": 11981, + "tgif": 14483, + "th": 513, + "th": 640, + "tha": 18470, + "tha": 4715, + "thab": 38219, + "thad": 48339, + "thai": 28054, + "thai": 8825, + "thail": 7258, + "thailand": 7469, + "thak": 22801, + "thakur": 38427, + "thal": 7967, + "thal": 12323, + "thala": 17784, + "thalai": 25206, + "thalaivar": 44918, + "thalap": 39789, + "thalapathy": 45405, + "thalapathy": 23324, + "thall": 36007, + "tham": 11761, + "tham": 8896, + "thames": 43472, + "thames": 15321, + "than": 792, + "than": 1126, + "thand": 44465, + "thane": 21463, + "thang": 24870, + "thani": 31322, + "thank": 2790, + "thank": 1144, + "thanked": 32079, + "thankful": 38839, + "thankful": 6217, + "thankfully": 22089, + "thanking": 21989, + "thanks": 5672, + "thanks": 1085, + "thanksgiving": 45732, + "thanksgiving": 6167, + "thanku": 45710, + "thankyou": 18050, + "thankyou": 9911, + "thanniversary": 35564, + "thanos": 36709, + "thanx": 25095, + "thar": 14396, + "thar": 38843, + "thard": 43474, + "that": 6303, + "that": 682, + "thatcher": 32496, + "thats": 44636, + "thats": 9254, + "thaw": 26081, + "thaw": 47229, + "thbewithyou": 41067, + "thc": 20091, + "thcentury": 49111, + "thd": 28219, + "thday": 37801, + "the": 599, + "the": 518, + "thea": 15935, + "thea": 25429, + "thead": 25259, + "theal": 45728, + "thealth": 31398, + "thear": 43283, + "theart": 44678, + "theast": 8378, + "theastern": 17877, + "theat": 2263, + "theater": 39438, + "theater": 6128, + "theaters": 14689, + "theatre": 19857, + "theatre": 3292, + "theatres": 21680, + "theatrical": 26833, + "theband": 27695, + "thebeatles": 35645, + "thebest": 40883, + "thebest": 25856, + "thebig": 24732, + "theblack": 47718, + "thec": 48659, + "thed": 31405, + "thedaily": 33550, + "theday": 4408, + "thedream": 39417, + "thee": 44475, + "thee": 15108, + "theeconomist": 44518, + "theellenshow": 35342, + "thefilm": 31665, + "theflash": 25434, + "theforce": 40002, + "theforceawakens": 48033, + "theft": 13286, + "thefuture": 34287, + "thegame": 24428, + "thegood": 28594, + "thegreat": 28721, + "thei": 44522, + "their": 911, + "theirs": 29297, + "thel": 5403, + "thelast": 23495, + "thelastjedi": 47992, + "theless": 27712, + "theli": 15277, + "thelittle": 46872, + "thelo": 47036, + "thelove": 40668, + "thelove": 43200, + "them": 5435, + "them": 1180, + "themasters": 48378, + "theme": 38524, + "theme": 5849, + "themed": 10126, + "themes": 17849, + "themet": 48183, + "themovie": 27062, + "themselves": 6503, + "then": 5929, + "then": 1594, + "thenburg": 45209, + "thene": 17012, + "thenew": 24212, + "thenext": 47881, + "thenight": 43336, + "theno": 37172, + "thenorth": 34338, + "theo": 17043, + "theo": 18084, + "theod": 26653, + "theodore": 30743, + "theological": 41162, + "theology": 24095, + "theon": 34653, + "theone": 46231, + "theopen": 41438, + "theore": 22690, + "theoretical": 35585, + "theori": 34804, + "theories": 23937, + "theory": 7143, + "thepeople": 33597, + "thepersonal": 29981, + "thepersonalnetwork": 30016, + "thephoto": 18303, + "thephotohour": 18607, + "ther": 1160, + "ther": 743, + "therap": 4499, + "therapeu": 19332, + "therapeutic": 23240, + "therapeutics": 49101, + "therapies": 30179, + "therapist": 20608, + "therapists": 34763, + "therapper": 49340, + "therapy": 5257, + "there": 5283, + "there": 997, + "thereal": 8074, + "thereal": 41140, + "thereby": 43308, + "thered": 10208, + "therefore": 16865, + "theres": 18494, + "theresa": 14126, + "therese": 47996, + "theresistance": 22845, + "theri": 28967, + "theri": 45297, + "therine": 26807, + "therine": 9239, + "thering": 7891, + "therland": 25351, + "thermal": 13689, + "thermo": 22303, + "thermom": 31138, + "thermometer": 38172, + "thermost": 42391, + "thern": 10919, + "thern": 3137, + "thero": 13165, + "theroad": 29807, + "therock": 30036, + "theroy": 38146, + "thers": 1959, + "thes": 40556, + "thes": 6460, + "thescript": 47061, + "these": 40366, + "these": 1071, + "theses": 39388, + "thesimpsons": 45513, + "thesims": 34192, + "thesis": 10673, + "thessal": 41491, + "thessaloni": 41753, + "thest": 35343, + "thesun": 45617, + "theta": 27694, + "thetic": 7954, + "thetimes": 36039, + "thevamp": 33701, + "thevoice": 47206, + "thevoice": 30258, + "thewalkingdead": 18087, + "thewanted": 43008, + "theworld": 44988, + "theworld": 17475, + "thex": 35990, + "they": 15174, + "they": 889, + "theyre": 28266, + "thfc": 17729, + "thi": 2362, + "thi": 9111, + "thia": 17943, + "thiago": 44537, + "thian": 23214, + "thians": 28187, + "thibau": 48351, + "thic": 26107, + "thic": 11794, + "thick": 18417, + "thick": 11006, + "thicker": 43302, + "thickness": 40754, + "thief": 18508, + "thier": 25595, + "thierry": 32929, + "thieves": 17899, + "thigh": 47124, + "thigh": 22877, + "thighs": 30847, + "thik": 20512, + "thika": 44619, + "thill": 31266, + "thim": 42331, + "thin": 2178, + "thin": 7847, + "thine": 47192, + "thing": 7499, + "thing": 946, + "things": 30670, + "things": 1739, + "thingsto": 43924, + "thingy": 36888, + "think": 9820, + "think": 1331, + "thinkbig": 26015, + "thinkbigsundaywithmarsha": 26666, + "thinker": 34577, + "thinkers": 32779, + "thinkin": 34443, + "thinking": 3291, + "thinks": 6109, + "thinner": 47247, + "thir": 6030, + "third": 32102, + "third": 3981, + "thirds": 42582, + "thirst": 23563, + "thirsty": 39731, + "thirsty": 17521, + "thirteen": 34209, + "thirty": 20813, + "thiru": 43292, + "this": 4340, + "this": 589, + "thisday": 6532, + "thisdayin": 33641, + "thisdayinhistory": 46913, + "thisi": 7299, + "thisis": 14887, + "thismorning": 36245, + "thistle": 29039, + "thistory": 28904, + "thium": 21804, + "thletics": 17765, + "thm": 10407, + "thman": 30079, + "thms": 19874, + "thn": 44155, + "thn": 45587, + "thnx": 25480, + "tho": 1325, + "tho": 5025, + "thof": 18943, + "thofjuly": 21613, + "thol": 29319, + "thole": 31029, + "tholes": 42465, + "thology": 9881, + "thom": 2585, + "thom": 24094, + "thomas": 12574, + "thomas": 3888, + "thome": 21289, + "thomp": 37274, + "thompson": 42181, + "thompson": 8535, + "thomson": 24151, + "thon": 38776, + "thon": 8924, + "thong": 37058, + "thood": 15623, + "thor": 4130, + "thor": 13691, + "thora": 46866, + "thorn": 12957, + "thorn": 18466, + "thorne": 18025, + "thorns": 33650, + "thornton": 23592, + "thorough": 15294, + "thorough": 34788, + "thoroughbred": 43248, + "thoroughly": 19750, + "thorpe": 18099, + "thos": 41965, + "those": 1753, + "thot": 33736, + "thou": 1513, + "thou": 17781, + "though": 2846, + "thought": 23948, + "thought": 2449, + "thoughtful": 19592, + "thoughts": 3618, + "thour": 27125, + "thousand": 9344, + "thousands": 7089, + "thouse": 40318, + "thouse": 7819, + "thoven": 23078, + "thr": 1111, + "thr": 19138, + "thra": 17761, + "thra": 32797, + "thrash": 38262, + "thre": 1607, + "thread": 31108, + "thread": 8815, + "threads": 24957, + "threat": 7527, + "threat": 7212, + "threaten": 26097, + "threatened": 16391, + "threatening": 16400, + "threatens": 20555, + "threats": 12766, + "three": 21615, + "three": 2097, + "thren": 41776, + "thresh": 29779, + "threshold": 33791, + "threw": 12746, + "thri": 8713, + "thrift": 27779, + "thrill": 21023, + "thrilled": 7879, + "thriller": 9653, + "thrilling": 20101, + "thrills": 39829, + "thrive": 17669, + "thriving": 22677, + "thro": 2101, + "thro": 28624, + "throat": 16371, + "thrombo": 47585, + "throne": 15999, + "thrones": 8072, + "throp": 34939, + "throttle": 37139, + "through": 6091, + "through": 1417, + "throughout": 6721, + "throughs": 48278, + "throw": 3315, + "throw": 6293, + "throwback": 6001, + "throwback": 5058, + "throwbackthursday": 6326, + "thrower": 40199, + "throwing": 9734, + "thrown": 15079, + "throws": 14723, + "thru": 23856, + "thru": 6162, + "thrush": 46133, + "thrust": 40202, + "ths": 2079, + "tht": 23554, + "thu": 3837, + "thu": 14153, + "thub": 25660, + "thug": 37212, + "thug": 18137, + "thugs": 27686, + "thul": 28368, + "thulhu": 37560, + "thum": 14679, + "thumb": 19514, + "thumb": 18674, + "thumbnail": 32365, + "thumbs": 17599, + "thun": 32267, + "thunder": 6161, + "thunder": 8951, + "thunderbird": 45131, + "thunderbirds": 44286, + "thunderbolt": 43596, + "thunderstorm": 12005, + "thunderstorms": 19525, + "thunt": 46763, + "thur": 1837, + "thur": 21704, + "thurman": 41291, + "thurs": 9908, + "thursday": 11218, + "thursday": 2221, + "thursdaymotivation": 39375, + "thursdays": 21444, + "thursdaythoughts": 14866, + "thurst": 33970, + "thus": 12457, + "thusi": 9488, + "thwaite": 48469, + "thweeksary": 30871, + "thx": 5913, + "thy": 7804, + "thy": 3362, + "thyme": 29805, + "thyro": 25174, + "thyroid": 32558, + "ti": 555, + "ti": 2605, + "tia": 6709, + "tial": 2826, + "tially": 14503, + "tian": 23011, + "tian": 8125, + "tians": 35182, + "tiara": 38322, + "tib": 47868, + "tibet": 19927, + "tibet": 22234, + "tibetan": 24057, + "tible": 11453, + "tic": 890, + "tic": 1550, + "tica": 9669, + "tical": 34191, + "tical": 4342, + "tically": 13375, + "ticals": 30861, + "tice": 3122, + "tich": 48769, + "tician": 43358, + "ticism": 26491, + "tick": 24640, + "tick": 15617, + "ticket": 25740, + "ticket": 4500, + "ticketing": 44432, + "tickets": 2015, + "ticking": 35842, + "tickle": 42999, + "ticks": 40269, + "tico": 17670, + "ticon": 45996, + "tics": 2419, + "ticul": 15538, + "ticus": 44277, + "tid": 26002, + "tid": 23727, + "tidal": 21949, + "tide": 15698, + "tide": 9105, + "tides": 25524, + "tidy": 23858, + "tie": 14072, + "tie": 3422, + "tied": 9889, + "tiem": 34762, + "tien": 47538, + "tiene": 43438, + "tier": 14390, + "tier": 6598, + "tierney": 45693, + "tiers": 24604, + "ties": 25556, + "ties": 2499, + "tiest": 18300, + "tiesto": 46367, + "tif": 23216, + "tiff": 11112, + "tiff": 20699, + "tiffany": 30467, + "tiffany": 14446, + "tification": 43923, + "tified": 40854, + "tiful": 29123, + "tify": 6677, + "tig": 31999, + "tiger": 11954, + "tiger": 6531, + "tigers": 6934, + "tigh": 31365, + "tight": 25763, + "tight": 9123, + "tighten": 46653, + "tighter": 48193, + "tightly": 37568, + "tights": 29581, + "tijuana": 45273, + "tik": 24986, + "tik": 32403, + "tiki": 30107, + "til": 6124, + "til": 1763, + "tile": 26217, + "tile": 8227, + "tiles": 10607, + "tility": 38180, + "till": 17462, + "till": 4267, + "tilla": 26063, + "tillerson": 47738, + "tilly": 41199, + "tilt": 23601, + "tim": 1292, + "tim": 3863, + "timate": 4754, + "timb": 26627, + "timber": 14441, + "timber": 16246, + "timberlake": 28274, + "timbers": 39911, + "timberwolves": 41190, + "time": 3764, + "time": 788, + "timed": 32727, + "timehop": 19944, + "timel": 23549, + "timelapse": 48154, + "timeless": 15558, + "timeline": 11492, + "timely": 19250, + "timeout": 41536, + "timer": 19725, + "timers": 44574, + "times": 26445, + "times": 1661, + "timesnow": 45487, + "timesof": 32522, + "timesofindia": 44182, + "timetable": 31971, + "timeto": 29187, + "timing": 13624, + "timm": 22444, + "timmy": 33252, + "timo": 13390, + "timo": 33777, + "timothy": 42087, + "timothy": 18560, + "timp": 42166, + "tin": 1310, + "tin": 5420, + "tina": 9257, + "tinder": 24287, + "tine": 22341, + "ting": 7451, + "ting": 694, + "tinged": 44829, + "tings": 35332, + "tini": 26839, + "tink": 39278, + "tinker": 45272, + "tinker": 40910, + "tino": 20538, + "tins": 37359, + "tint": 40497, + "tinted": 42618, + "tiny": 21716, + "tiny": 5591, + "tio": 27562, + "tion": 2274, + "tion": 740, + "tional": 22460, + "tional": 2986, + "tionality": 24514, + "tionally": 12409, + "tionary": 8381, + "tione": 44318, + "tioned": 9083, + "tioning": 15528, + "tionist": 25732, + "tions": 1371, + "tious": 14255, + "tip": 15383, + "tip": 4623, + "tipoff": 44521, + "tipp": 32294, + "tipped": 31878, + "tipper": 38095, + "tipperary": 45612, + "tipping": 27827, + "tips": 3173, + "tipton": 48809, + "tiptuesday": 42112, + "tique": 37772, + "tir": 25467, + "tir": 38462, + "tire": 29128, + "tire": 9362, + "tired": 6533, + "tireless": 39835, + "tirelessly": 41548, + "tires": 15533, + "tiring": 42630, + "tiru": 36033, + "tis": 7839, + "tis": 7394, + "tise": 13745, + "tisgarh": 40538, + "tish": 45148, + "tish": 28784, + "tism": 27113, + "tiss": 28155, + "tissue": 15368, + "tissues": 32172, + "tist": 7902, + "tista": 25580, + "tists": 25944, + "tit": 1991, + "tit": 13202, + "tita": 40936, + "titan": 13496, + "titan": 15516, + "titanic": 20729, + "titanium": 24409, + "titans": 13066, + "titi": 17434, + "titi": 48504, + "title": 28033, + "title": 3644, + "titled": 9939, + "titles": 9780, + "tito": 26838, + "titus": 36102, + "tium": 21975, + "tiv": 1835, + "tiva": 41886, + "tive": 14640, + "tive": 1420, + "tively": 9883, + "tiveness": 20955, + "tives": 7570, + "tivity": 9859, + "tivo": 32162, + "tix": 5835, + "tiz": 19376, + "tj": 18890, + "tj": 18988, + "tk": 22344, + "tk": 20676, + "tko": 37347, + "tks": 38739, + "tl": 14325, + "tl": 8190, + "tland": 30697, + "tlap": 41976, + "tlc": 22047, + "tle": 39141, + "tle": 5825, + "tles": 39363, + "tless": 17427, + "tlot": 41080, + "tls": 47367, + "tly": 37483, + "tly": 1646, + "tm": 9430, + "tm": 7789, + "tman": 20796, + "tmc": 35263, + "tment": 26485, + "tml": 39445, + "tmltalk": 42260, + "tmnt": 32444, + "tmobile": 34901, + "tmr": 35906, + "tmrw": 16496, + "tms": 44496, + "tmund": 23801, + "tmw": 45827, + "tmz": 37248, + "tn": 3827, + "tn": 7248, + "tna": 21150, + "tnam": 8079, + "tner": 34922, + "tness": 35212, + "tney": 9523, + "tng": 35898, + "tnt": 20659, + "tnx": 38220, + "to": 580, + "to": 531, + "toa": 17916, + "toad": 26096, + "toast": 24654, + "toast": 10920, + "toasted": 23533, + "toaster": 39061, + "toasty": 44726, + "tob": 24260, + "tobac": 12611, + "tobacco": 13905, + "tobago": 39482, + "tobe": 17534, + "tobe": 28740, + "tober": 18162, + "tober": 2925, + "toberfest": 26249, + "tobi": 40335, + "tobi": 48374, + "tobias": 32464, + "tobin": 42466, + "toby": 29659, + "toby": 18333, + "toc": 41907, + "toc": 30643, + "tock": 25274, + "tod": 38239, + "tod": 33568, + "toda": 47141, + "todas": 36150, + "today": 11800, + "today": 721, + "todayin": 32957, + "todays": 13513, + "todayshow": 29739, + "todd": 10398, + "todd": 9951, + "toddler": 17772, + "toddlers": 36719, + "toddy": 38926, + "todo": 48857, + "todo": 23087, + "todos": 33355, + "toe": 47756, + "toe": 11344, + "toes": 16511, + "tof": 6659, + "toff": 27319, + "toffee": 34880, + "tofficial": 47953, + "tofthe": 23678, + "toftheday": 20566, + "tofu": 24692, + "tog": 45715, + "toge": 1903, + "together": 17858, + "together": 1952, + "togo": 26729, + "tography": 33968, + "toh": 26851, + "toi": 7472, + "toi": 26941, + "toid": 49124, + "toile": 43148, + "toilet": 11071, + "toilets": 24027, + "toire": 39534, + "tok": 16690, + "tok": 27010, + "token": 32634, + "token": 17134, + "tokens": 23562, + "tokyo": 35038, + "tokyo": 6667, + "tol": 4678, + "tol": 32962, + "told": 3527, + "tole": 15677, + "toledo": 19812, + "toler": 12150, + "tolerance": 20377, + "tolerant": 38536, + "tolerate": 35556, + "tolkien": 32989, + "toll": 44090, + "toll": 14155, + "tollywood": 42016, + "tology": 34799, + "tom": 999, + "tom": 2435, + "toma": 42360, + "toma": 44710, + "tomas": 35944, + "tomas": 27178, + "tomat": 12041, + "tomato": 9867, + "tomatoes": 13004, + "tomb": 37187, + "tomb": 15582, + "tombs": 48613, + "tombstone": 45729, + "tome": 24137, + "tome": 24283, + "tomi": 46290, + "tomlin": 46649, + "tomlinson": 17484, + "tommorow": 42871, + "tommy": 16573, + "tommy": 8876, + "tomo": 31223, + "tomo": 34434, + "tomor": 1277, + "tomorrow": 19728, + "tomorrow": 1293, + "tomorrowland": 34951, + "tomorrows": 32258, + "tomorrowspaper": 35005, + "tomorrowspaperstoday": 35190, + "tomp": 43544, + "tompkins": 49068, + "toms": 10545, + "tomy": 18730, + "ton": 838, + "ton": 917, + "tona": 13459, + "tone": 32366, + "tone": 8408, + "toned": 29426, + "toner": 40614, + "tones": 14744, + "tong": 21510, + "tonga": 37882, + "tongue": 44820, + "tongue": 13626, + "tongues": 39837, + "toni": 17766, + "toni": 17171, + "tonic": 17808, + "tonics": 34647, + "tonight": 1009, + "tonights": 23312, + "tonite": 13449, + "tonka": 42781, + "tonline": 45867, + "tonne": 42450, + "tonnes": 24813, + "tons": 7555, + "tony": 9150, + "tony": 4767, + "tonyawards": 46068, + "too": 1843, + "too": 1256, + "took": 2280, + "tool": 13718, + "tool": 5999, + "toolbox": 46599, + "toolkit": 29849, + "tools": 5771, + "toom": 27550, + "toon": 24664, + "toon": 19701, + "toonami": 48336, + "toons": 35345, + "toor": 42590, + "tooth": 15316, + "tooth": 12030, + "toothbrush": 36841, + "toothpaste": 37322, + "tooting": 42969, + "top": 5534, + "top": 1253, + "topaz": 46125, + "tope": 32149, + "tope": 42239, + "topeka": 46884, + "topia": 29618, + "topic": 8720, + "topical": 37464, + "topics": 11916, + "topless": 37415, + "topo": 23008, + "topoli": 30152, + "topp": 19529, + "topped": 12588, + "topper": 31780, + "toppers": 41651, + "topping": 21071, + "toppings": 47554, + "topps": 20201, + "tops": 8154, + "topshop": 40953, + "topus": 21495, + "tor": 937, + "tor": 1208, + "tora": 45147, + "torah": 37945, + "toral": 45282, + "torch": 31921, + "torch": 15820, + "tore": 38066, + "tore": 19385, + "tored": 38046, + "torg": 33214, + "tori": 17689, + "tori": 17539, + "toria": 23732, + "torial": 28029, + "torian": 48399, + "tories": 14193, + "torino": 29178, + "torio": 34235, + "torn": 8572, + "torn": 18023, + "tornad": 24676, + "tornado": 9062, + "tornadoes": 28254, + "toro": 17892, + "toron": 37407, + "toronto": 16866, + "toronto": 4514, + "torpe": 34093, + "torpedo": 46582, + "torquay": 45738, + "torque": 31940, + "torre": 39563, + "torre": 38009, + "torrent": 42317, + "torrential": 41158, + "torres": 16049, + "tors": 2546, + "tortilla": 32683, + "torto": 24170, + "tortoise": 30178, + "torture": 16013, + "tortured": 29900, + "tory": 29390, + "tory": 4214, + "tos": 6094, + "tosc": 37719, + "tose": 38154, + "tosh": 17109, + "toshi": 31744, + "toss": 19656, + "tossed": 31296, + "tot": 4618, + "tot": 23659, + "total": 13507, + "total": 4445, + "totally": 5440, + "totals": 25772, + "tote": 48145, + "tote": 19031, + "totem": 45376, + "totes": 37199, + "tothe": 12222, + "toto": 39823, + "tots": 24978, + "totten": 14360, + "tottenham": 14889, + "tou": 1879, + "tou": 29261, + "touch": 9480, + "touch": 4526, + "touchdown": 18664, + "touchdowns": 37905, + "touched": 13190, + "touches": 14832, + "touching": 14088, + "touchscreen": 39095, + "tough": 12063, + "tough": 5499, + "tougher": 33722, + "toughest": 23773, + "toughness": 45522, + "toulou": 27145, + "toulouse": 30267, + "tour": 2710, + "tour": 1760, + "tourde": 39247, + "toured": 27654, + "touri": 4224, + "touring": 11853, + "tourism": 23661, + "tourism": 6556, + "tourist": 12123, + "tourists": 15546, + "tournament": 4097, + "tournaments": 23058, + "tourney": 12603, + "tours": 8948, + "tous": 37424, + "tout": 22300, + "touts": 41274, + "tov": 28970, + "tow": 11557, + "tow": 18653, + "toward": 8508, + "towards": 4447, + "towed": 45419, + "towel": 15953, + "towels": 26578, + "tower": 26669, + "tower": 4730, + "towering": 39444, + "towers": 12701, + "towie": 44613, + "towin": 45819, + "towing": 36963, + "town": 4068, + "town": 1605, + "townfc": 33981, + "townhall": 33408, + "townhouse": 40178, + "towns": 14173, + "townsend": 26826, + "township": 14622, + "townsville": 47330, + "towork": 48233, + "tox": 7742, + "tox": 16145, + "toxic": 27436, + "toxic": 12348, + "toxicity": 41234, + "toxin": 48899, + "toxins": 36618, + "toy": 14387, + "toy": 5988, + "toya": 37602, + "toyo": 7644, + "toyota": 8908, + "toys": 39508, + "toys": 7162, + "tp": 23760, + "tp": 15188, + "tpp": 29411, + "tps": 35246, + "tq": 43066, + "tr": 635, + "tr": 6337, + "tra": 752, + "tra": 2483, + "trac": 2266, + "trace": 48611, + "trace": 14767, + "traced": 47956, + "traces": 30913, + "tracey": 25558, + "tracing": 27897, + "track": 10887, + "track": 2700, + "tracked": 27049, + "tracker": 18123, + "tracking": 10428, + "tracklist": 39777, + "tracks": 7579, + "tract": 4690, + "traction": 10644, + "tractor": 14607, + "tractors": 37854, + "tracy": 32984, + "tracy": 15508, + "trad": 48716, + "trad": 38037, + "trade": 10457, + "trade": 3629, + "traded": 18860, + "trademark": 25011, + "trader": 17700, + "traders": 19112, + "trades": 18519, + "trading": 40083, + "trading": 6520, + "tradio": 20689, + "tradition": 20838, + "tradition": 8784, + "traditional": 41113, + "traditional": 5604, + "traditionally": 35532, + "traditions": 18016, + "traf": 3227, + "trafal": 32461, + "trafalgar": 36969, + "traff": 31571, + "traffic": 12080, + "traffic": 3399, + "trafficking": 15983, + "trafford": 22912, + "trage": 12430, + "tragedy": 14082, + "tragic": 14828, + "tragically": 39599, + "trail": 11523, + "trail": 4921, + "trailblazer": 41015, + "trailblazers": 35954, + "trailer": 4700, + "trailers": 24862, + "trailing": 37427, + "trails": 10633, + "train": 9122, + "train": 3231, + "trained": 10874, + "trainee": 25795, + "trainees": 30382, + "trainer": 9767, + "trainers": 18871, + "training": 34508, + "training": 2199, + "trains": 9541, + "trait": 35160, + "traitor": 31760, + "traitors": 42633, + "traits": 25748, + "trajec": 42042, + "trak": 24065, + "tral": 14609, + "tram": 9800, + "tram": 17500, + "tramp": 46289, + "trampol": 32905, + "trampoline": 42800, + "tramrahim": 35220, + "tran": 1357, + "tran": 22031, + "trance": 30584, + "trance": 18671, + "trancefamily": 39630, + "trane": 35779, + "tranqu": 18912, + "tranquil": 35764, + "tranquility": 36688, + "trans": 1826, + "trans": 8126, + "transaction": 24881, + "transactions": 21653, + "transat": 37872, + "transatlantic": 40703, + "transc": 21073, + "transcend": 47087, + "transcript": 39008, + "transcription": 48765, + "transfer": 22659, + "transfer": 7134, + "transferred": 29700, + "transferring": 40924, + "transfers": 21621, + "transform": 8142, + "transform": 12288, + "transformation": 34204, + "transformation": 7832, + "transformational": 47135, + "transformationtuesday": 36511, + "transformative": 38106, + "transformed": 17453, + "transformer": 38235, + "transformers": 17843, + "transforming": 44470, + "transforming": 19251, + "transforms": 30312, + "transgender": 17732, + "transi": 32236, + "transit": 10174, + "transiti": 22939, + "transition": 11391, + "transitional": 41519, + "transitioning": 43586, + "transitions": 39374, + "transl": 12243, + "translate": 22655, + "translated": 20752, + "translates": 36334, + "translating": 42156, + "translation": 12153, + "translations": 41367, + "translator": 36230, + "translucent": 49052, + "transm": 18861, + "transmission": 16103, + "transmitted": 48605, + "transmitter": 40457, + "transp": 11726, + "transpa": 18524, + "transparen": 16108, + "transparency": 16828, + "transparent": 19017, + "transpl": 16038, + "transplant": 41871, + "transplant": 18771, + "transplantation": 45207, + "transpor": 19406, + "transport": 10231, + "transport": 7362, + "transportation": 10911, + "transported": 29089, + "transporter": 43568, + "transporting": 42259, + "trap": 36224, + "trap": 9677, + "trape": 42435, + "trapped": 15592, + "traps": 28517, + "tras": 30638, + "trash": 39215, + "trash": 9798, + "traum": 22263, + "trauma": 13846, + "traumati": 46613, + "traumatic": 29958, + "trav": 7586, + "trav": 46955, + "trave": 35357, + "travel": 2824, + "travel": 1949, + "travelblog": 35957, + "travelblogger": 25494, + "travelchat": 46455, + "traveled": 20384, + "traveler": 17794, + "travelers": 20644, + "travelgram": 40069, + "traveling": 9365, + "travelled": 23428, + "traveller": 22546, + "travellers": 29583, + "travelling": 11190, + "travelphotography": 22808, + "travelpics": 32293, + "travels": 11472, + "traveltips": 36260, + "traveltuesday": 16713, + "traverse": 35058, + "travi": 46971, + "travis": 27441, + "travis": 12287, + "traw": 42288, + "trax": 34421, + "tray": 38470, + "tray": 14621, + "trays": 39798, + "trc": 41803, + "tre": 975, + "tre": 6033, + "treach": 46005, + "tread": 26182, + "tread": 35658, + "treadmill": 37780, + "treas": 8591, + "treason": 28103, + "treasure": 9922, + "treasured": 48068, + "treasurer": 26985, + "treasures": 16500, + "treasury": 20956, + "treat": 3968, + "treat": 3901, + "treated": 9772, + "treating": 13842, + "treatment": 4869, + "treatments": 15839, + "treats": 8878, + "treaty": 19967, + "treble": 33194, + "trecht": 33812, + "tree": 13354, + "tree": 2677, + "treehouse": 42387, + "trees": 4682, + "trek": 13236, + "trek": 8136, + "trekking": 25293, + "trell": 35159, + "tremb": 44043, + "tremend": 14659, + "tremendous": 15988, + "tren": 2579, + "trench": 23846, + "trenches": 38723, + "trend": 19986, + "trend": 6643, + "trending": 6087, + "trends": 7015, + "trendsetter": 46666, + "trendy": 23072, + "trent": 45885, + "trent": 15548, + "trenton": 37470, + "tres": 23569, + "tress": 4733, + "tresses": 24273, + "trevor": 23437, + "trevor": 13219, + "trex": 42114, + "trey": 36670, + "trey": 16939, + "tri": 924, + "tri": 9618, + "triad": 45602, + "trial": 5991, + "trials": 10992, + "triangle": 14615, + "triathlon": 18080, + "trib": 45151, + "tribal": 16629, + "tribe": 19943, + "tribe": 11365, + "tribeca": 35184, + "tribes": 26546, + "tribu": 3028, + "tribun": 14311, + "tribunal": 32911, + "tribune": 18556, + "tribute": 5493, + "tributes": 15537, + "tric": 9511, + "tric": 4081, + "trich": 39519, + "trick": 17177, + "trick": 8172, + "tricks": 13177, + "tricky": 22319, + "trics": 31437, + "trident": 35491, + "tridge": 18722, + "tried": 4554, + "tries": 4315, + "trife": 48962, + "trigge": 30509, + "trigger": 16158, + "triggered": 30924, + "triggers": 37319, + "tright": 29915, + "tril": 40626, + "trill": 39297, + "trilli": 39350, + "trillion": 20160, + "trilo": 15183, + "trilogy": 16862, + "trim": 14182, + "trimmed": 40657, + "trin": 6628, + "trinidad": 26244, + "trinity": 30744, + "trinity": 12267, + "trio": 10263, + "trip": 23421, + "trip": 2529, + "tripad": 37189, + "tripadvisor": 38708, + "triple": 16519, + "triple": 7673, + "triplets": 48601, + "tripod": 36141, + "tripoli": 40095, + "trippin": 43073, + "tripping": 35229, + "trippy": 35137, + "trips": 12292, + "tris": 29690, + "trish": 40511, + "trish": 37179, + "trisha": 39152, + "tristan": 25497, + "trit": 37087, + "triton": 45437, + "triu": 14782, + "trium": 21065, + "triumph": 26507, + "triumph": 15307, + "triumphant": 41918, + "trivi": 21228, + "trivia": 10642, + "triviatuesday": 45499, + "trix": 41017, + "tro": 1046, + "tro": 3332, + "trock": 44368, + "trojan": 30653, + "trojans": 25310, + "trol": 10306, + "troll": 39737, + "troll": 17103, + "trolley": 25124, + "trolling": 28552, + "trolls": 20890, + "tromb": 32390, + "trombone": 44423, + "tron": 19057, + "tron": 10684, + "tronic": 34258, + "tronics": 34397, + "troom": 23691, + "troop": 12492, + "troop": 24054, + "trooper": 18327, + "troopers": 23576, + "troops": 10109, + "trop": 31585, + "trope": 41150, + "trophies": 20998, + "trophy": 42676, + "trophy": 6502, + "tropic": 21794, + "tropic": 36736, + "tropical": 41699, + "tropical": 8686, + "tropics": 36940, + "tros": 40456, + "trose": 36022, + "trot": 30453, + "trotter": 38287, + "trou": 5181, + "troubad": 49037, + "trouble": 25669, + "trouble": 7848, + "troubled": 25568, + "troubles": 27254, + "trough": 39761, + "troupe": 34803, + "trous": 19727, + "trousers": 23172, + "trout": 14853, + "trove": 45350, + "trow": 46914, + "troy": 26283, + "troy": 12819, + "trs": 24770, + "tru": 931, + "tru": 25326, + "truck": 14781, + "truck": 4629, + "trucker": 45918, + "truckers": 43404, + "trucking": 26208, + "trucks": 9569, + "trude": 39017, + "trudeau": 15752, + "true": 13096, + "true": 2328, + "truec": 37583, + "truelove": 45711, + "truffle": 23064, + "truffles": 37057, + "truly": 4545, + "trum": 11766, + "trum": 11399, + "truman": 29414, + "trump": 9124, + "trump": 1797, + "trumpet": 23681, + "trumpp": 45550, + "trumprussia": 39135, + "trumps": 29793, + "trumptrain": 43595, + "trun": 16163, + "trun": 46661, + "trunk": 18347, + "trunks": 38531, + "truro": 43507, + "truss": 46080, + "trust": 17691, + "trust": 3876, + "truste": 17356, + "trusted": 16538, + "trustee": 30803, + "trustees": 28853, + "trusting": 33221, + "trusts": 27507, + "trustworthy": 46840, + "trusty": 37955, + "truth": 21335, + "truth": 4319, + "truths": 27179, + "trx": 31620, + "try": 4487, + "try": 1209, + "tryin": 31085, + "trying": 2551, + "tryna": 15702, + "tryout": 43832, + "tryouts": 28053, + "ts": 2290, + "ts": 590, + "tsa": 25977, + "tsal": 20438, + "tsb": 45015, + "tsc": 37437, + "tsch": 38778, + "tsd": 20611, + "tse": 49144, + "tsfor": 42654, + "tsford": 32823, + "tsh": 42872, + "tshirt": 14907, + "tshirts": 29377, + "tsi": 40048, + "tsi": 37867, + "tsk": 43600, + "tsla": 35681, + "tsm": 43452, + "tsman": 20046, + "tsn": 44921, + "tsn": 26896, + "tson": 42353, + "tson": 47140, + "tsp": 34230, + "tsu": 13950, + "tsu": 20175, + "tsun": 19155, + "tsunami": 24286, + "tsville": 29080, + "tt": 971, + "tt": 1402, + "tta": 2646, + "ttc": 27668, + "tte": 23105, + "tte": 3070, + "tted": 15163, + "tten": 11351, + "tten": 17479, + "tter": 18691, + "tter": 5165, + "tters": 6318, + "ttes": 9293, + "tti": 5237, + "ttin": 36589, + "tting": 1188, + "ttino": 47389, + "ttip": 46993, + "ttle": 9253, + "ttm": 46838, + "tto": 8759, + "tto": 8105, + "tton": 10562, + "ttot": 12480, + "ttp": 30828, + "ttr": 47589, + "tts": 11570, + "ttt": 17256, + "tttt": 33119, + "ttu": 44006, + "ttv": 24281, + "tty": 11457, + "tty": 1856, + "tu": 764, + "tu": 5760, + "tua": 41344, + "tual": 4799, + "tuan": 37297, + "tub": 34907, + "tub": 15450, + "tube": 38229, + "tube": 3308, + "tuber": 30371, + "tuberculo": 42606, + "tuberculosis": 43129, + "tubes": 22870, + "tubing": 40794, + "tubs": 41705, + "tubular": 48786, + "tuc": 14456, + "tuc": 43871, + "tuck": 22398, + "tucked": 26923, + "tucker": 39703, + "tucker": 15726, + "tucket": 32677, + "tucson": 17250, + "tudor": 24547, + "tue": 17515, + "tues": 2283, + "tues": 12113, + "tuesday": 10209, + "tuesday": 2519, + "tuesdaymotivation": 25432, + "tuesdays": 23195, + "tuesdaythoughts": 17988, + "tuf": 44510, + "tuff": 38868, + "tug": 47032, + "tug": 27902, + "tuition": 21129, + "tuk": 39271, + "tuk": 14993, + "tul": 9069, + "tul": 40837, + "tula": 36332, + "tulane": 44893, + "tulip": 28389, + "tulips": 30886, + "tulsa": 18850, + "tum": 12932, + "tum": 8843, + "tumb": 8831, + "tumble": 38284, + "tumbler": 48790, + "tumbling": 46226, + "tumblr": 11841, + "tummy": 26053, + "tumor": 22616, + "tumors": 39894, + "tumour": 45129, + "tun": 1415, + "tun": 21349, + "tuna": 15037, + "tundra": 39899, + "tune": 11427, + "tune": 3300, + "tuned": 5898, + "tunein": 16809, + "tuner": 42905, + "tunes": 31688, + "tunes": 10810, + "tunesapp": 32550, + "tung": 47940, + "tung": 31092, + "tuni": 16270, + "tunic": 43495, + "tuning": 19585, + "tunisia": 23346, + "tunnel": 11096, + "tunnels": 29814, + "tuous": 28738, + "tup": 37956, + "tup": 4507, + "tupac": 31506, + "tups": 44855, + "tur": 985, + "tur": 17182, + "tura": 16127, + "tural": 45143, + "tural": 4261, + "turb": 18973, + "turban": 48515, + "turbine": 26880, + "turbines": 38863, + "turbo": 23578, + "turbo": 13668, + "turbul": 31100, + "turbulent": 47871, + "ture": 4321, + "ture": 941, + "tured": 3987, + "turer": 11993, + "turers": 16956, + "tures": 2400, + "turf": 36762, + "turf": 12510, + "turi": 11896, + "turin": 36251, + "turing": 5812, + "turismo": 30202, + "turk": 8254, + "turk": 32507, + "turkey": 35977, + "turkey": 4790, + "turkeys": 37991, + "turkish": 48199, + "turkish": 9278, + "turks": 34344, + "turmeric": 34044, + "turmoil": 37751, + "turn": 5522, + "turn": 2105, + "turnaround": 32719, + "turnbull": 27863, + "turned": 3771, + "turner": 42867, + "turner": 8777, + "turning": 4976, + "turno": 21377, + "turnout": 11654, + "turnover": 30794, + "turnpike": 38301, + "turns": 3185, + "turnt": 28887, + "turntable": 37953, + "turnup": 30591, + "turo": 29224, + "turquo": 19390, + "turquoise": 19899, + "turt": 13716, + "turtle": 35943, + "turtle": 10912, + "turtles": 17862, + "tus": 24828, + "tus": 7079, + "tusc": 17909, + "tuscal": 42638, + "tuscaloosa": 44375, + "tuscan": 42865, + "tuscany": 20885, + "tuss": 31741, + "tut": 35121, + "tutor": 10054, + "tutor": 27858, + "tutorial": 12857, + "tutorials": 30973, + "tutoring": 37532, + "tutti": 46880, + "tutu": 35845, + "tux": 28720, + "tux": 49186, + "tuxedo": 40173, + "tv": 3197, + "tv": 1583, + "tvc": 49190, + "tvd": 25889, + "tvmiaw": 38554, + "tvn": 44232, + "tvs": 27114, + "tvtime": 19947, + "tvxq": 43968, + "tw": 966, + "tw": 12842, + "twa": 46954, + "twain": 30689, + "twal": 48126, + "tware": 5707, + "twc": 41217, + "twd": 29440, + "twd": 19343, + "twdfamily": 38218, + "twe": 18365, + "tweak": 48870, + "tweaks": 42661, + "twee": 1330, + "tweed": 26904, + "tweeps": 14928, + "tweet": 11826, + "tweet": 1842, + "tweeta": 32024, + "tweetapicture": 40596, + "tweeted": 7841, + "tweeter": 32876, + "tweeters": 31713, + "tweeting": 8901, + "tweets": 3560, + "tweetyour": 45033, + "twel": 14476, + "twelf": 39443, + "twelfth": 44072, + "twell": 38722, + "twell": 30162, + "twelve": 19694, + "twent": 27027, + "twenti": 35167, + "twenty": 13016, + "twentyon": 39609, + "twentyonepilots": 40007, + "twer": 13923, + "twerk": 28506, + "twi": 5537, + "twice": 6970, + "twick": 34326, + "twickenham": 39619, + "twil": 12804, + "twili": 35754, + "twilight": 46366, + "twilight": 14512, + "twill": 43703, + "twin": 9342, + "twin": 6769, + "twine": 42775, + "twinkle": 36545, + "twinning": 30156, + "twinpeaks": 32042, + "twins": 8040, + "twist": 10589, + "twisted": 18233, + "twister": 45933, + "twists": 34149, + "twit": 1643, + "twit": 18704, + "twitart": 27709, + "twitch": 13251, + "twitch": 9153, + "twitter": 7546, + "twitter": 1989, + "twitterkurds": 32722, + "twitterstorians": 35389, + "two": 17211, + "two": 1237, + "twol": 31964, + "twood": 40404, + "twood": 13245, + "twp": 33283, + "twright": 46778, + "twt": 6825, + "twx": 26830, + "twy": 45861, + "tx": 6636, + "tx": 5200, + "txhsfb": 34757, + "txlege": 26995, + "txst": 40761, + "txt": 24595, + "txwx": 22995, + "ty": 1260, + "ty": 744, + "tya": 41273, + "tycoon": 36803, + "tye": 43097, + "tyfree": 41215, + "tyga": 41952, + "tying": 22559, + "tyl": 47537, + "tyler": 14787, + "tyler": 7058, + "tym": 45772, + "tyne": 27000, + "tyne": 29729, + "tyour": 16823, + "type": 15673, + "type": 3877, + "typed": 40753, + "typeface": 44969, + "types": 7543, + "typewriter": 42180, + "typho": 17486, + "typhoon": 21110, + "typic": 21648, + "typical": 9854, + "typically": 23175, + "typing": 20102, + "typo": 18831, + "typo": 29076, + "typography": 24332, + "tyr": 15590, + "tyran": 46921, + "tyranny": 35402, + "tyre": 38330, + "tyre": 16864, + "tyres": 21376, + "tyrone": 30226, + "tyson": 16616, + "tz": 7710, + "tz": 4983, + "tzer": 45267, + "tzky": 47127, + "tzman": 46032, + "tzu": 34354, + "té": 27208, + "té": 39694, + "u": 84, + "u": 340, + "ua": 34075, + "ua": 8441, + "uaap": 46753, + "uaap": 43774, + "uab": 35587, + "uae": 9752, + "ual": 1921, + "ually": 10767, + "uan": 33062, + "uas": 38339, + "uav": 30303, + "ub": 18430, + "ub": 13494, + "uba": 29768, + "ubc": 42479, + "ubc": 29455, + "ube": 30892, + "uber": 25896, + "uber": 10668, + "ubi": 26758, + "ubio": 32867, + "ubiquit": 48129, + "ubis": 28248, + "ubisoft": 32051, + "ubs": 43851, + "ubun": 28184, + "ubuntu": 30791, + "uc": 4903, + "uc": 12438, + "uca": 30942, + "ucc": 44844, + "ucc": 29138, + "ucci": 30746, + "uccino": 30409, + "ucd": 44746, + "ucd": 43514, + "ucf": 24414, + "uch": 19465, + "uch": 22394, + "uchi": 37473, + "uci": 46354, + "uci": 28925, + "uck": 34189, + "ucl": 12013, + "ucl": 13647, + "ucla": 37667, + "ucla": 17259, + "ucn": 49036, + "uconn": 30549, + "ud": 6560, + "ud": 5765, + "uda": 22800, + "udaipur": 49385, + "uddin": 43035, + "ude": 37016, + "ude": 35194, + "ue": 16696, + "ue": 1190, + "uefa": 19189, + "uel": 24231, + "uer": 45951, + "ues": 2526, + "uf": 17777, + "uf": 19230, + "ufc": 20396, + "ufc": 6490, + "uff": 45701, + "ufo": 19443, + "ufos": 48234, + "ug": 3754, + "ug": 16061, + "uga": 16056, + "ugand": 25965, + "uganda": 11125, + "ugandan": 44206, + "ugby": 30658, + "ugh": 39736, + "ugh": 12755, + "ugliest": 43543, + "ugly": 36070, + "ugly": 8159, + "ugu": 18144, + "uh": 17661, + "uh": 9219, + "uhc": 44974, + "uhh": 35938, + "uhhh": 45270, + "uhm": 35614, + "uhur": 29434, + "uhuru": 35690, + "ui": 17326, + "ui": 11458, + "uil": 29395, + "uit": 30696, + "uit": 47584, + "uj": 33266, + "uji": 39672, + "uk": 2294, + "uk": 1432, + "uka": 23294, + "uke": 48836, + "uke": 28577, + "uked": 48987, + "uki": 37435, + "uki": 9009, + "ukin": 34996, + "ukip": 20360, + "uklabour": 36902, + "ukmfg": 38764, + "uko": 33562, + "ukone": 24682, + "ukrain": 15468, + "ukraine": 7768, + "ukrainian": 16927, + "ukrunchat": 34481, + "uku": 29541, + "uku": 36082, + "ukulele": 39094, + "ul": 914, + "ul": 6625, + "ula": 34104, + "ula": 9506, + "ular": 4927, + "ulary": 21701, + "ulate": 20467, + "ulation": 32896, + "ule": 35616, + "ules": 26274, + "ulf": 49331, + "uli": 41841, + "uli": 22174, + "ull": 33254, + "ulla": 30577, + "ullah": 45310, + "ullivan": 45252, + "ulls": 37418, + "ulo": 46084, + "ulo": 36738, + "ulous": 42490, + "ulous": 4281, + "ulously": 20167, + "ulster": 29709, + "ulster": 24639, + "ult": 4380, + "ulti": 11925, + "ulties": 21884, + "ultimat": 16522, + "ultimate": 34684, + "ultimate": 5377, + "ultimatefan": 48372, + "ultimatefanlive": 48644, + "ultimately": 23023, + "ultr": 25636, + "ultra": 11398, + "ultra": 8118, + "ultram": 44519, + "ultrasound": 29717, + "ulture": 22272, + "ulty": 8036, + "ulu": 41815, + "ulu": 15659, + "ulum": 17235, + "uly": 33220, + "ulysses": 46114, + "um": 1622, + "um": 1008, + "uma": 29982, + "uma": 9256, + "uman": 27112, + "umar": 25656, + "umass": 39390, + "umatic": 45006, + "umb": 7493, + "umber": 19195, + "umbrel": 34773, + "umbrella": 17143, + "umbrellas": 42782, + "umbria": 39287, + "umc": 39491, + "umd": 42067, + "ume": 38480, + "umen": 42832, + "uments": 25924, + "umer": 23539, + "umes": 21403, + "umi": 48772, + "umi": 15458, + "umich": 41294, + "umin": 31542, + "umm": 26129, + "umm": 21215, + "ummer": 47628, + "ummm": 33665, + "umni": 31739, + "ump": 22224, + "umpire": 36214, + "ums": 8643, + "umu": 39788, + "un": 569, + "un": 2271, + "una": 6385, + "unable": 17793, + "unacceptable": 25234, + "unanim": 20800, + "unanimous": 33520, + "unanimously": 31798, + "unanswered": 43611, + "unarmed": 41541, + "unas": 41366, + "unavailable": 48430, + "unaware": 33347, + "unbeat": 37056, + "unbeatable": 40267, + "unbeaten": 19228, + "unbeliev": 11383, + "unbelievable": 13306, + "unbelievably": 33781, + "unborn": 37257, + "unboxing": 32866, + "unbreakable": 32956, + "unbroken": 49271, + "unc": 24921, + "unc": 15322, + "uncanny": 32556, + "uncertain": 30384, + "uncertainty": 23956, + "unch": 1527, + "unchanged": 34272, + "uncharted": 34560, + "unci": 25521, + "unciation": 34117, + "uncle": 31537, + "uncle": 8002, + "unclear": 32955, + "uncles": 45335, + "uncomfortable": 22470, + "uncommon": 34888, + "uncondition": 46561, + "unconditional": 31112, + "unconscious": 34791, + "unconstitutional": 43585, + "unconventional": 39440, + "uncover": 33031, + "uncovered": 28234, + "uncture": 38736, + "uncut": 41056, + "und": 9762, + "und": 9732, + "unda": 39932, + "undant": 25377, + "unday": 29338, + "unde": 45226, + "undead": 40105, + "undecided": 49368, + "undefeated": 15326, + "undeni": 38424, + "under": 1473, + "under": 1798, + "underage": 45669, + "underattack": 35075, + "undercover": 21595, + "underdog": 44266, + "undere": 21675, + "underestim": 23348, + "underestimate": 31794, + "undergo": 31545, + "undergoing": 26419, + "undergrad": 38331, + "undergraduate": 24320, + "underground": 9396, + "undering": 30826, + "underlying": 31812, + "undermine": 42839, + "underneath": 20857, + "underrated": 19494, + "unders": 20376, + "understand": 47582, + "understand": 4600, + "understanding": 7522, + "understands": 21607, + "understatement": 38296, + "understood": 17303, + "undertaker": 40144, + "undertaking": 49067, + "undertale": 48283, + "underthe": 41161, + "underwater": 14760, + "underway": 6273, + "underwear": 21154, + "underwood": 21474, + "underworld": 34760, + "undi": 23845, + "undisclosed": 39334, + "undo": 35454, + "undocumented": 35414, + "undoub": 38836, + "undoubtedly": 42204, + "undp": 26691, + "une": 4522, + "une": 10966, + "unearth": 32716, + "unearthed": 36632, + "unemp": 15139, + "unemployed": 32721, + "unemployment": 19350, + "unes": 6394, + "unesco": 16216, + "uneven": 43204, + "unex": 9484, + "unexpe": 10802, + "unexpec": 31829, + "unexpected": 12293, + "unexpectedly": 35622, + "unf": 29285, + "unfair": 22193, + "unfinished": 26526, + "unfit": 45367, + "unfold": 38681, + "unfollow": 38797, + "unfor": 14010, + "unforgettable": 16173, + "unfortun": 10194, + "unfortunate": 22361, + "unfortunately": 12863, + "unfpa": 45048, + "ung": 10439, + "ung": 4334, + "unga": 19151, + "ungsoo": 25582, + "unh": 25365, + "unhappy": 26528, + "unhcr": 43451, + "unhealthy": 30994, + "uni": 1107, + "uni": 5926, + "unic": 7648, + "unicef": 38286, + "unicef": 19259, + "unicorn": 15660, + "unicorns": 35183, + "unidenti": 33707, + "unidentified": 35563, + "unification": 45036, + "unified": 20876, + "uniform": 11075, + "uniforms": 17838, + "unil": 32388, + "unilever": 48654, + "uniof": 21218, + "union": 14210, + "union": 3503, + "unions": 18353, + "unis": 30482, + "unis": 39266, + "unisex": 27609, + "unison": 46694, + "unit": 28522, + "unit": 5695, + "unite": 15078, + "unite": 11305, + "uniteblue": 20935, + "united": 10898, + "united": 2690, + "unitedstates": 39636, + "unitedway": 47486, + "unites": 32061, + "uniting": 31318, + "units": 10394, + "unity": 38300, + "unity": 8581, + "univ": 36680, + "univ": 14896, + "univer": 15574, + "univers": 5855, + "universal": 19148, + "universal": 8754, + "universe": 6104, + "universi": 41692, + "universit": 26019, + "universities": 16408, + "university": 40728, + "university": 2182, + "universityof": 46158, + "unk": 5542, + "unknown": 8685, + "unl": 43807, + "unlawful": 42305, + "unle": 19677, + "unlea": 23893, + "unleash": 26706, + "unleashed": 27955, + "unless": 10602, + "unlike": 16694, + "unlikely": 18904, + "unlimited": 11015, + "unlock": 18649, + "unlocked": 16770, + "unlocking": 40810, + "unlucky": 35029, + "unlv": 42283, + "unmanned": 36751, + "unmatched": 46054, + "unn": 38364, + "unnamed": 44985, + "unnecessary": 24100, + "unner": 31481, + "unning": 43282, + "unnoticed": 42807, + "uno": 32446, + "uno": 17078, + "unofficial": 22506, + "unpacking": 43589, + "unpaid": 32811, + "unparalleled": 44396, + "unplugged": 31724, + "unpopular": 40232, + "unprece": 23054, + "unprecedented": 23344, + "unpredictable": 38684, + "unra": 45150, + "unreal": 46980, + "unreal": 15636, + "unrelated": 38644, + "unreleased": 29654, + "unrest": 36452, + "uns": 25908, + "unsafe": 32071, + "unsc": 36395, + "unseen": 19069, + "unsigned": 39346, + "unsolved": 40836, + "unsplash": 46196, + "unstable": 34730, + "unstopp": 22105, + "unstoppable": 23484, + "unsuccessful": 47478, + "unsung": 33015, + "unsure": 26396, + "unt": 19654, + "unt": 6537, + "until": 1942, + "untitled": 21309, + "unto": 19801, + "untold": 32206, + "untouch": 44509, + "untouched": 42764, + "unused": 29636, + "unusual": 12613, + "unusually": 36465, + "unve": 6685, + "unveil": 20483, + "unveiled": 13572, + "unveiling": 20327, + "unveils": 15057, + "unwanted": 25285, + "unwind": 34064, + "unya": 37142, + "uo": 30874, + "uo": 36162, + "uof": 11155, + "uoft": 37329, + "uon": 48144, + "uous": 40185, + "up": 1083, + "up": 705, + "upa": 31727, + "upbeat": 39201, + "upcoming": 4196, + "upcycled": 46552, + "upd": 3226, + "update": 2491, + "updated": 5974, + "updates": 4904, + "updating": 22792, + "uper": 38082, + "uper": 33056, + "upfront": 42064, + "upgrade": 10365, + "upgraded": 18577, + "upgrades": 21253, + "upgrading": 34368, + "uph": 14128, + "uphill": 42767, + "uphol": 26195, + "uphold": 43897, + "upholstery": 44556, + "upl": 41939, + "uplift": 45389, + "uplifting": 29546, + "upload": 13968, + "uploaded": 16793, + "uploading": 30145, + "upon": 23524, + "upon": 5067, + "upp": 19549, + "upp": 45946, + "upper": 22465, + "upper": 7067, + "upri": 15982, + "upright": 29818, + "uprising": 26006, + "upro": 28922, + "ups": 6926, + "upscale": 47501, + "upset": 11214, + "upsets": 42637, + "upside": 15362, + "upstairs": 21387, + "upstate": 33335, + "upstream": 45517, + "upthe": 31510, + "upto": 26575, + "upton": 31910, + "uptown": 23807, + "upward": 32526, + "upwards": 34915, + "uq": 39591, + "ur": 565, + "ur": 1775, + "ura": 29337, + "ura": 3544, + "urable": 40194, + "ural": 23547, + "ural": 33948, + "uran": 16197, + "uranium": 29850, + "urban": 7931, + "urban": 5800, + "urbanart": 40834, + "urd": 47880, + "urday": 19742, + "urdu": 29976, + "ure": 5514, + "ure": 726, + "ured": 4210, + "urer": 20864, + "ures": 2288, + "urg": 35995, + "urge": 14852, + "urged": 23790, + "urgency": 47612, + "urgent": 13693, + "urgently": 34534, + "urges": 16692, + "urging": 27748, + "uri": 11052, + "uri": 8699, + "urie": 46429, + "urin": 45245, + "urine": 28864, + "uring": 1351, + "url": 23464, + "urn": 38075, + "uro": 17343, + "uro": 5925, + "urology": 48585, + "urope": 14918, + "urs": 4794, + "urself": 31942, + "urst": 19181, + "urstruly": 34751, + "urstrulymahesh": 35314, + "ursula": 38390, + "urt": 24309, + "uru": 16322, + "uru": 11768, + "uruguay": 27931, + "urus": 14246, + "urve": 24583, + "ury": 8642, + "ury": 2106, + "us": 904, + "us": 718, + "usa": 9491, + "usa": 2547, + "usability": 46736, + "usable": 22890, + "usaf": 25017, + "usage": 19137, + "usaid": 34507, + "usair": 36742, + "usairforce": 42179, + "usarmy": 19132, + "usatoday": 40263, + "usav": 36056, + "usb": 10281, + "usc": 13346, + "usc": 14995, + "uscg": 43932, + "usd": 7485, + "usda": 25829, + "use": 4419, + "use": 1483, + "used": 32289, + "used": 2026, + "useful": 9784, + "useless": 20154, + "usemb": 39700, + "user": 21248, + "user": 7031, + "username": 28162, + "users": 7433, + "uses": 5282, + "useum": 45189, + "usf": 32385, + "usf": 28942, + "usgs": 35103, + "ush": 12001, + "ush": 18335, + "usher": 27411, + "ushi": 47734, + "usi": 25540, + "usic": 34909, + "usic": 16753, + "using": 1996, + "usky": 45778, + "usl": 42113, + "usm": 40041, + "usmc": 21678, + "usmnt": 30662, + "usn": 40579, + "usnavy": 24500, + "usnews": 43752, + "uso": 21539, + "usopen": 21782, + "usp": 26651, + "usps": 39980, + "usrc": 33274, + "uss": 11545, + "uss": 9260, + "ussia": 29553, + "ussoccer": 42828, + "ussr": 32697, + "ust": 35501, + "ust": 24725, + "usu": 4254, + "usu": 40434, + "usual": 6129, + "usually": 8296, + "usur": 45582, + "uswnt": 35255, + "ut": 1419, + "ut": 3641, + "uta": 42706, + "uta": 25925, + "utah": 27474, + "utah": 9312, + "utc": 18196, + "utd": 10493, + "ute": 16856, + "ute": 3130, + "uten": 32089, + "uter": 39197, + "utes": 2850, + "uth": 48819, + "uth": 44750, + "uti": 24568, + "util": 28824, + "utili": 17015, + "utilities": 27210, + "utility": 14941, + "utilize": 36861, + "utilized": 47604, + "utilizing": 40212, + "utm": 47853, + "utmost": 42352, + "uto": 18866, + "uto": 13683, + "utopia": 34433, + "utpol": 42605, + "utr": 48726, + "utrecht": 37216, + "uts": 11740, + "utsa": 37528, + "utt": 17096, + "uttar": 40168, + "uttarak": 33755, + "uttarakhand": 35655, + "utter": 18769, + "utter": 24558, + "utterly": 21353, + "utto": 42183, + "utv": 36351, + "utz": 45320, + "uu": 5702, + "uu": 14553, + "uuu": 44355, + "uuu": 27656, + "uuuu": 16720, + "uuuu": 40797, + "uv": 23777, + "uv": 15977, + "uva": 23908, + "uw": 13933, + "uw": 19166, + "uwe": 48785, + "uwu": 35544, + "ux": 9251, + "ux": 6213, + "uy": 31929, + "uy": 48113, + "uz": 19398, + "uz": 36991, + "uzbe": 43007, + "uzbekistan": 45024, + "uzzi": 48210, + "v": 85, + "v": 341, + "va": 4648, + "va": 1892, + "vaa": 37488, + "vable": 23088, + "vac": 3125, + "vac": 34085, + "vaca": 48215, + "vacancies": 26333, + "vacancy": 21247, + "vacant": 25262, + "vacation": 28336, + "vacation": 6561, + "vacations": 29002, + "vacay": 44716, + "vacc": 13342, + "vaccin": 19164, + "vaccinated": 48134, + "vaccination": 32518, + "vaccine": 47780, + "vaccine": 17493, + "vaccines": 25860, + "vach": 46211, + "vacu": 16058, + "vacuum": 18420, + "vad": 11880, + "vada": 46759, + "vader": 21908, + "vae": 39384, + "vag": 13015, + "vague": 42154, + "vah": 26921, + "vai": 26893, + "vai": 36802, + "vail": 21189, + "vain": 25538, + "vais": 28719, + "vaj": 34206, + "vak": 16288, + "vak": 41597, + "val": 1214, + "val": 1560, + "vala": 48525, + "valdez": 40617, + "vale": 35554, + "vale": 10820, + "valedic": 43525, + "valen": 12630, + "valence": 30225, + "valenci": 34183, + "valencia": 16559, + "valent": 3655, + "valent": 15300, + "valentin": 48631, + "valentina": 43741, + "valentine": 11208, + "valentine": 5876, + "valentines": 10259, + "valentinesday": 12369, + "valentino": 29624, + "valeri": 31951, + "valerie": 25592, + "valet": 45749, + "vali": 8230, + "valiant": 33804, + "valid": 15126, + "validation": 32536, + "valkyrie": 42326, + "vall": 23523, + "vall": 35295, + "vallarta": 47874, + "valle": 24857, + "valle": 29105, + "valley": 18354, + "valley": 3136, + "valleys": 28649, + "valor": 30930, + "vals": 7431, + "valu": 6291, + "valuable": 10056, + "valuation": 25894, + "value": 41358, + "value": 4602, + "valued": 17801, + "values": 8857, + "valve": 17001, + "valves": 33517, + "vam": 9983, + "vamo": 46718, + "vamos": 30346, + "vamp": 10680, + "vampi": 47017, + "vampire": 47576, + "vampire": 13220, + "vampires": 30868, + "vamps": 44810, + "van": 2446, + "van": 2451, + "vana": 20543, + "vanc": 6320, + "vance": 31447, + "vancou": 6750, + "vancouver": 31904, + "vancouver": 7208, + "vand": 11691, + "vandalism": 45664, + "vander": 16264, + "vanderbilt": 33524, + "vandy": 39268, + "vane": 43828, + "vaness": 13328, + "vanessa": 16836, + "vangogh": 47849, + "vanguard": 27916, + "vani": 15396, + "vani": 26459, + "vania": 10998, + "vanilla": 11974, + "vanished": 43783, + "vanishing": 48296, + "vanity": 48353, + "vanity": 22938, + "vans": 11711, + "vant": 26298, + "vantage": 31749, + "vanu": 42892, + "vanuatu": 48766, + "vap": 10462, + "vape": 25423, + "vape": 20219, + "vaping": 29403, + "vapor": 37167, + "vapor": 30729, + "vapori": 46183, + "var": 3187, + "var": 12998, + "vara": 47492, + "varan": 36585, + "varanasi": 39364, + "vard": 21866, + "vard": 8773, + "vardy": 47371, + "vare": 38159, + "vares": 42895, + "vargas": 32752, + "vari": 3354, + "variable": 26416, + "varian": 34334, + "variant": 20293, + "variants": 38312, + "variation": 26420, + "variations": 29025, + "varied": 32334, + "varies": 32543, + "varieties": 23805, + "variety": 8396, + "various": 7395, + "varsity": 43716, + "varsity": 8574, + "varun": 48120, + "varun": 22069, + "vary": 18855, + "varying": 36456, + "vas": 5669, + "vas": 5995, + "vasc": 40995, + "vascular": 19218, + "vase": 20431, + "vasi": 49092, + "vast": 24413, + "vast": 16414, + "vastly": 48257, + "vat": 11588, + "vat": 18363, + "vatican": 21030, + "vation": 37884, + "vau": 6391, + "vaugh": 25158, + "vaughan": 21392, + "vaughn": 29013, + "vaul": 27469, + "vault": 15240, + "vaus": 40217, + "vaux": 27403, + "vauxhall": 29173, + "vaw": 47952, + "vay": 48000, + "vaz": 38142, + "vb": 29365, + "vb": 8778, + "vball": 38329, + "vc": 28670, + "vc": 7952, + "vcs": 43528, + "vcu": 40102, + "vd": 9515, + "vday": 42055, + "ve": 673, + "ve": 563, + "vea": 43798, + "veal": 36616, + "veau": 24419, + "vec": 19912, + "vector": 40453, + "vector": 21533, + "ved": 19515, + "ved": 1102, + "veda": 44401, + "vedere": 45660, + "vedi": 47971, + "vee": 35708, + "vee": 17073, + "veen": 22432, + "veer": 21243, + "veer": 22058, + "veg": 9048, + "veg": 16460, + "vega": 22930, + "vegan": 15705, + "vegan": 5615, + "vegans": 48514, + "vegas": 20288, + "vegas": 4413, + "vege": 6219, + "vegetable": 15725, + "vegetables": 14119, + "vegetarian": 14600, + "vegetation": 33947, + "veggie": 19401, + "veggies": 16767, + "vehic": 3973, + "vehicle": 5299, + "vehicles": 8361, + "veil": 23516, + "vein": 29169, + "veins": 28867, + "veit": 30620, + "vel": 942, + "vel": 1287, + "vela": 34898, + "veld": 34011, + "veled": 15370, + "veli": 49166, + "veling": 37970, + "vell": 21173, + "vell": 32997, + "velo": 14357, + "velo": 33850, + "velocity": 23811, + "vels": 5109, + "velve": 37849, + "velvet": 11063, + "vely": 1708, + "vember": 3477, + "vement": 3129, + "vements": 11104, + "ven": 1240, + "ven": 1638, + "vena": 47442, + "vend": 10851, + "vending": 29202, + "vendor": 21261, + "vendors": 20353, + "vene": 5365, + "veness": 10516, + "venetian": 34336, + "venezia": 34139, + "venezu": 10939, + "venezuela": 12839, + "venezuelan": 34699, + "veng": 31526, + "venge": 27757, + "vengeance": 32057, + "veni": 31142, + "venice": 11010, + "vening": 47532, + "venison": 40037, + "venom": 42491, + "venom": 21588, + "vens": 20884, + "vent": 4373, + "vent": 5687, + "ventil": 39522, + "ventilation": 35066, + "venting": 15731, + "vention": 4122, + "vents": 12833, + "ventu": 48217, + "ventura": 20921, + "venture": 37046, + "venture": 12543, + "ventures": 20829, + "venue": 5097, + "venues": 18120, + "venus": 14691, + "ver": 624, + "ver": 667, + "vera": 13350, + "verage": 3725, + "verb": 34952, + "verbal": 26522, + "verbally": 39985, + "verbs": 45687, + "verde": 16935, + "verdi": 42306, + "verdict": 18030, + "vere": 11135, + "vere": 34707, + "vered": 2868, + "verge": 23913, + "veri": 11638, + "verification": 33521, + "verified": 22555, + "verify": 34722, + "vering": 4630, + "veriz": 19707, + "verizon": 21532, + "verma": 41261, + "vermont": 19241, + "vern": 2214, + "vern": 12586, + "verne": 45553, + "vernon": 18348, + "vero": 45217, + "vero": 38208, + "verona": 31819, + "veronic": 39551, + "veronica": 24039, + "vers": 1219, + "vers": 2094, + "versa": 35765, + "versace": 25422, + "versail": 29857, + "versailles": 32129, + "versary": 2940, + "versatile": 18110, + "versatility": 41340, + "verse": 39466, + "verse": 3131, + "verses": 30769, + "versi": 8934, + "version": 3273, + "versions": 16190, + "versity": 1906, + "verst": 42484, + "verstappen": 45064, + "versus": 14548, + "versy": 18522, + "vert": 11742, + "verte": 35158, + "verted": 48173, + "verti": 30459, + "vertical": 14293, + "vertigo": 42477, + "verton": 40632, + "verts": 37265, + "very": 11698, + "very": 1070, + "veryday": 37944, + "verything": 45174, + "ves": 9616, + "ves": 1003, + "vesmatter": 47636, + "vespa": 46029, + "vessel": 16387, + "vessels": 22822, + "vest": 31657, + "vest": 12473, + "vesti": 40349, + "vests": 41906, + "vet": 12294, + "vet": 5951, + "veter": 4330, + "veteran": 20797, + "veteran": 8814, + "veterans": 7092, + "veteransday": 26409, + "veterin": 43959, + "veterinary": 25458, + "veto": 36570, + "vets": 13113, + "vette": 17045, + "vettel": 28700, + "vevo": 35141, + "vex": 36187, + "vex": 43978, + "vey": 34792, + "vey": 3884, + "vez": 35987, + "vez": 17226, + "vf": 25966, + "vfl": 33726, + "vfx": 30149, + "vg": 40591, + "vg": 22346, + "vh": 46953, + "vh": 23847, + "vhs": 21932, + "vi": 603, + "vi": 4259, + "via": 1048, + "viable": 25752, + "viadu": 37012, + "viaduct": 39113, + "vial": 39951, + "vian": 40487, + "vian": 16124, + "vibe": 37974, + "vibe": 12813, + "vibes": 7764, + "vibr": 9527, + "vibrant": 14270, + "vibration": 37456, + "vibrations": 43660, + "vic": 1555, + "vic": 4412, + "vica": 46168, + "vicar": 43899, + "vice": 43572, + "vice": 6931, + "vicente": 39411, + "vices": 8332, + "vich": 24143, + "vici": 46670, + "vicious": 25177, + "vick": 15116, + "vick": 29704, + "vickers": 48452, + "vicki": 34927, + "vicky": 37176, + "vicky": 25788, + "victi": 6861, + "victim": 9133, + "victims": 7131, + "victor": 2423, + "victor": 10690, + "victori": 17555, + "victoria": 39286, + "victoria": 6127, + "victorian": 12350, + "victorias": 47791, + "victories": 24577, + "victorious": 24033, + "victory": 36668, + "victory": 4127, + "vid": 17233, + "vid": 9284, + "vida": 19015, + "vidal": 36678, + "vide": 1334, + "vide": 45244, + "video": 9478, + "video": 1455, + "videogame": 35097, + "videogames": 21149, + "videos": 6081, + "vids": 23035, + "vidy": 29639, + "vidya": 45264, + "vie": 922, + "vie": 8538, + "vien": 36493, + "vienna": 12670, + "vier": 15352, + "vier": 11987, + "viera": 21114, + "viernes": 33826, + "vies": 22458, + "viest": 31979, + "viet": 17558, + "viet": 13128, + "vietnam": 19558, + "vietnam": 8623, + "vietnamese": 22382, + "view": 12004, + "view": 1093, + "viewed": 7226, + "viewer": 15061, + "viewers": 14275, + "viewing": 7124, + "viewpoint": 41604, + "views": 2758, + "vig": 8549, + "vig": 45083, + "vigil": 21538, + "vigil": 19896, + "vigilant": 43026, + "vigne": 40447, + "vigne": 34581, + "vigo": 44097, + "vigor": 26781, + "vii": 17759, + "viii": 20414, + "vijay": 12014, + "vijay": 10823, + "vijaysethu": 47966, + "vik": 10764, + "vik": 17181, + "vika": 39562, + "vikas": 37116, + "viking": 26663, + "viking": 15897, + "vikings": 11713, + "vikram": 41136, + "vikram": 24314, + "viktor": 36101, + "vil": 1338, + "vil": 3000, + "vila": 37505, + "vile": 27247, + "vill": 10481, + "vill": 45698, + "villa": 3203, + "villa": 7754, + "village": 34584, + "village": 4331, + "villagers": 34283, + "villages": 17621, + "villain": 15425, + "villains": 25271, + "villanova": 44025, + "villar": 35164, + "villas": 28907, + "ville": 11110, + "ville": 1930, + "villen": 46177, + "villi": 36907, + "vimeo": 48720, + "vin": 1379, + "vin": 2558, + "vina": 35682, + "vinai": 37396, + "vinaigrette": 39876, + "vinay": 43952, + "vince": 32429, + "vince": 6236, + "vincen": 33402, + "vincent": 29069, + "vincent": 10357, + "vinci": 30199, + "vind": 20275, + "vindic": 39582, + "vine": 8471, + "vine": 7721, + "vinegar": 23834, + "vines": 21268, + "vineyard": 16527, + "vineyards": 23082, + "ving": 5375, + "ving": 903, + "vingne": 42579, + "vings": 22510, + "vini": 48119, + "vinnie": 40885, + "vinny": 36794, + "vino": 14509, + "vinod": 43348, + "vins": 34820, + "vinson": 45945, + "vintag": 10936, + "vintage": 13654, + "vintage": 3266, + "viny": 40990, + "vinyl": 22835, + "vinyl": 5754, + "vio": 11913, + "vio": 20324, + "viol": 3164, + "viola": 27438, + "violate": 44875, + "violated": 38192, + "violating": 37554, + "violation": 22919, + "violations": 21969, + "violence": 5450, + "violent": 11565, + "violently": 47758, + "violet": 16118, + "violets": 42861, + "violin": 17058, + "violinist": 36299, + "vion": 35496, + "vious": 6418, + "viously": 7149, + "vip": 45714, + "vip": 7111, + "viper": 27401, + "vips": 41149, + "vir": 1790, + "vir": 25319, + "vira": 35910, + "viral": 11653, + "virat": 32473, + "virgil": 39076, + "virgin": 5651, + "virgin": 12103, + "virgini": 43426, + "virginia": 6728, + "virgo": 39978, + "viro": 32301, + "viron": 38309, + "virtu": 7977, + "virtual": 18059, + "virtual": 7790, + "virtually": 22475, + "virtualreality": 32608, + "virtue": 26860, + "virtues": 42167, + "virtuoso": 47027, + "virus": 11808, + "viruses": 34830, + "vis": 1301, + "vis": 5337, + "visa": 12802, + "visas": 41228, + "vise": 24977, + "vised": 14810, + "vish": 12024, + "vish": 29124, + "vishal": 33648, + "vishnu": 37816, + "visi": 1409, + "visibility": 15921, + "visible": 36658, + "visible": 8626, + "vising": 37439, + "vision": 11147, + "vision": 2515, + "visional": 24627, + "visionary": 22959, + "visions": 13804, + "visit": 3388, + "visit": 1600, + "visitation": 44370, + "visited": 5580, + "visiting": 4680, + "visitor": 13881, + "visitors": 9160, + "visits": 8489, + "visitscotland": 28760, + "visitspain": 48860, + "vism": 15514, + "viso": 46732, + "visor": 24217, + "vist": 21436, + "vista": 13865, + "visu": 7739, + "visual": 17004, + "visual": 7195, + "visualization": 28500, + "visualize": 45057, + "visually": 25743, + "visuals": 21315, + "viswas": 36513, + "viswasam": 47664, + "vit": 4056, + "vit": 35580, + "vita": 15700, + "vital": 32525, + "vital": 10585, + "vitality": 36385, + "vitam": 9856, + "vitamin": 13675, + "vitamins": 22582, + "vito": 36725, + "vity": 4893, + "vitz": 26188, + "vius": 41571, + "viv": 21827, + "viv": 35363, + "viva": 17399, + "vival": 35920, + "vive": 18980, + "vive": 24004, + "vivek": 36243, + "vivi": 11625, + "vivian": 30129, + "vivid": 22984, + "vivo": 28091, + "vivo": 25888, + "vix": 28976, + "vix": 34811, + "vixen": 38757, + "vixx": 32106, + "viz": 28251, + "viz": 31786, + "vj": 45439, + "vj": 30827, + "vk": 41893, + "vl": 37580, + "vl": 36442, + "vla": 23686, + "vlad": 41089, + "vladi": 19320, + "vladimir": 21702, + "vlive": 46797, + "vlog": 18894, + "vm": 16204, + "vm": 20269, + "vma": 35666, + "vmas": 30236, + "vmware": 29615, + "vn": 47098, + "vn": 25076, + "vo": 947, + "vo": 3951, + "voc": 4105, + "voc": 20855, + "vocab": 21346, + "vocabulary": 23804, + "vocal": 34037, + "vocal": 13147, + "vocali": 19134, + "vocalist": 22102, + "vocals": 17666, + "vocation": 20521, + "vocational": 33751, + "vod": 11820, + "vod": 35854, + "vodaf": 28436, + "vodafone": 38695, + "vodka": 13646, + "vogel": 44960, + "vogue": 24418, + "vogue": 13178, + "voic": 29185, + "voice": 13179, + "voice": 3386, + "voiced": 34352, + "voiceof": 44966, + "voiceover": 41979, + "voices": 9144, + "void": 21561, + "voip": 42762, + "voir": 16036, + "vol": 1343, + "vol": 7945, + "volatile": 41022, + "volatility": 32355, + "volcan": 9916, + "volcanic": 24072, + "volcano": 14581, + "volcanoes": 38055, + "voli": 40138, + "volk": 13432, + "volkswag": 14407, + "volkswagen": 15342, + "volley": 7130, + "volley": 34656, + "volleyball": 7458, + "volo": 44791, + "vols": 20404, + "volt": 26430, + "volta": 29879, + "volta": 33480, + "voltage": 23118, + "voltron": 39314, + "volu": 3563, + "volume": 8284, + "volumes": 22651, + "volun": 3356, + "voluntar": 48823, + "voluntary": 23815, + "volunte": 3556, + "volunteer": 32331, + "volunteer": 7114, + "volunteered": 34000, + "volunteering": 14902, + "volunteers": 5939, + "volution": 24043, + "volved": 42888, + "volvo": 39991, + "volvo": 16906, + "vom": 24198, + "vomit": 46485, + "von": 11269, + "von": 8497, + "voo": 19497, + "voodoo": 26869, + "voor": 34291, + "voor": 34464, + "vor": 8338, + "vor": 5308, + "vore": 18215, + "vortex": 30071, + "vos": 16863, + "vot": 48558, + "vote": 6830, + "vote": 2187, + "voted": 6454, + "votel": 41379, + "voter": 44474, + "voter": 14065, + "voters": 8925, + "votes": 6693, + "voting": 5756, + "vou": 11045, + "voucher": 18190, + "vouchers": 23384, + "vous": 10636, + "vow": 34787, + "vows": 21677, + "vox": 29215, + "vox": 22692, + "voy": 10622, + "voy": 15021, + "voyage": 16299, + "voyager": 29669, + "vp": 32758, + "vp": 3896, + "vpn": 38212, + "vr": 16840, + "vr": 5921, + "vre": 44500, + "vre": 17501, + "vs": 11385, + "vs": 1547, + "vsco": 26752, + "vsco": 32822, + "vscocam": 34694, + "vsky": 37791, + "vss": 31919, + "vt": 31732, + "vt": 10291, + "vu": 8664, + "vu": 13230, + "vue": 43915, + "vue": 19313, + "vuel": 31312, + "vuelta": 43856, + "vuitton": 26705, + "vul": 6856, + "vulcan": 34767, + "vulner": 11213, + "vulnerability": 28797, + "vulnerable": 14332, + "vulture": 34593, + "vultures": 47197, + "vv": 19264, + "vv": 35686, + "vw": 28650, + "vw": 13250, + "vx": 47644, + "vy": 11566, + "vy": 5157, + "w": 86, + "w": 342, + "wa": 869, + "wa": 2663, + "waa": 35874, + "wab": 19893, + "wab": 36852, + "wac": 27445, + "wac": 37947, + "wack": 22880, + "wack": 38270, + "wacky": 34318, + "waco": 36035, + "wad": 11133, + "wad": 30451, + "wada": 40006, + "wade": 40237, + "wade": 14180, + "wadi": 37253, + "waf": 17638, + "wafc": 49086, + "waff": 13940, + "waffle": 20375, + "waffles": 24205, + "wag": 5764, + "wag": 19177, + "wage": 10716, + "wager": 43430, + "wages": 19114, + "wagner": 18081, + "wagon": 13260, + "wagons": 47944, + "wags": 48580, + "wah": 24812, + "wah": 18014, + "wahl": 27500, + "wahlberg": 35151, + "wahoo": 47995, + "wai": 11469, + "wai": 21569, + "waifu": 46551, + "waikiki": 44907, + "wain": 28358, + "wain": 20120, + "wainwright": 45878, + "waist": 36946, + "waist": 18459, + "wait": 10021, + "wait": 1885, + "waite": 24272, + "waited": 18492, + "waiter": 32946, + "waitin": 44482, + "waiting": 2680, + "waitress": 39760, + "waitrose": 37164, + "waits": 21361, + "waiver": 42866, + "waj": 49367, + "wak": 11172, + "wak": 36015, + "waka": 42696, + "wake": 10501, + "wake": 5731, + "wakefield": 26358, + "wakes": 29108, + "wakeup": 26328, + "wakeup": 35380, + "wakeupamerica": 37474, + "waking": 13025, + "wal": 1056, + "wal": 6903, + "wala": 16468, + "walang": 49180, + "walcott": 45744, + "wald": 46930, + "wald": 15724, + "walden": 39311, + "waldo": 32440, + "waldorf": 38227, + "wale": 41247, + "wale": 20336, + "wales": 25383, + "wales": 5110, + "walgreens": 38490, + "wali": 37576, + "wali": 14768, + "walia": 44455, + "walk": 8588, + "walk": 2374, + "walkaway": 48255, + "walked": 8667, + "walker": 24735, + "walker": 6150, + "walkers": 23366, + "walkin": 45792, + "walking": 12644, + "walking": 3941, + "walkingdead": 14948, + "walkout": 47470, + "walks": 8192, + "walkway": 36614, + "wall": 4316, + "wall": 2569, + "walla": 26007, + "walla": 39982, + "wallabies": 48926, + "wallace": 12535, + "wallart": 36223, + "walled": 36567, + "waller": 45340, + "wallet": 12154, + "wallets": 38550, + "walleye": 49099, + "wallis": 42206, + "wallpaper": 10560, + "wallpapers": 29841, + "walls": 8258, + "wallstreet": 45341, + "wally": 26024, + "walmart": 11972, + "walnut": 16310, + "walnuts": 38294, + "walsall": 42935, + "walsh": 12856, + "walt": 23535, + "walt": 14312, + "waltdisneyworld": 36505, + "walter": 31156, + "walter": 10645, + "walters": 25532, + "waltham": 42742, + "waltham": 45581, + "walton": 19485, + "waltz": 35982, + "wam": 20503, + "wamy": 46970, + "wan": 2060, + "wan": 4557, + "wana": 30830, + "wand": 14636, + "wand": 28559, + "wanda": 25070, + "wander": 12985, + "wander": 24473, + "wandered": 46593, + "wanderers": 27540, + "wandering": 22597, + "wanderlust": 16129, + "wane": 27459, + "wang": 19731, + "wang": 11900, + "wani": 21674, + "wankers": 42189, + "wann": 23622, + "wanna": 35940, + "wanna": 3836, + "wannabe": 40730, + "wannaone": 44832, + "want": 18356, + "want": 1280, + "wanted": 3146, + "wanting": 12801, + "wants": 3107, + "wap": 27393, + "wap": 30368, + "waq": 47512, + "war": 984, + "war": 2238, + "wara": 21631, + "warbler": 33891, + "warcraft": 13660, + "ward": 7728, + "ward": 1460, + "warden": 27798, + "wardly": 30780, + "wardro": 14247, + "wardrobe": 15020, + "wards": 2593, + "ware": 7416, + "ware": 4476, + "wareagle": 35716, + "warehouse": 13054, + "wareness": 41601, + "wareness": 35870, + "wares": 30692, + "warfare": 15739, + "warhammer": 26832, + "warhol": 27554, + "wari": 20977, + "wark": 46346, + "wark": 15164, + "warlock": 42455, + "warm": 14725, + "warm": 3616, + "warmed": 36695, + "warmer": 14328, + "warmest": 30910, + "warming": 8606, + "warmly": 45322, + "warmongers": 33205, + "warms": 32917, + "warmth": 19636, + "warmup": 29904, + "warmups": 44094, + "warn": 19360, + "warned": 16409, + "warner": 28564, + "warner": 13402, + "warning": 4994, + "warnings": 18098, + "warns": 14086, + "waron": 38947, + "warp": 32411, + "warped": 32125, + "warran": 17392, + "warrant": 22554, + "warrants": 45677, + "warranty": 23999, + "warren": 23143, + "warren": 9234, + "warri": 4109, + "warrington": 31203, + "warrior": 18998, + "warrior": 8148, + "warriors": 6421, + "wars": 3931, + "warsaw": 21072, + "warship": 47846, + "wart": 43535, + "wart": 7346, + "wartime": 42998, + "warts": 21781, + "warwick": 23081, + "warwick": 22215, + "warwickshire": 36766, + "wary": 36213, + "was": 3398, + "was": 739, + "wasabi": 47334, + "wash": 3363, + "wash": 7810, + "washed": 14092, + "washer": 24085, + "washes": 38950, + "washing": 13029, + "washington": 16774, + "washington": 4365, + "washingtondc": 40225, + "washingtonpost": 28426, + "wasn": 5044, + "wasnt": 29607, + "wasp": 24889, + "wasps": 35300, + "wassup": 45708, + "wast": 28886, + "waste": 18157, + "waste": 6065, + "wasted": 18278, + "wasteland": 44035, + "wastewater": 34463, + "wasting": 25577, + "wat": 800, + "wat": 10621, + "wata": 42509, + "watch": 7046, + "watch": 1239, + "watchdog": 35303, + "watched": 5775, + "watcher": 35971, + "watchers": 28443, + "watches": 9521, + "watchin": 32432, + "watching": 2113, + "water": 2505, + "water": 1573, + "watercolor": 14211, + "watercolour": 18377, + "waterfall": 16403, + "waterfalls": 26692, + "waterford": 24448, + "waterfront": 16605, + "waterhouse": 45072, + "watering": 19871, + "waterloo": 17465, + "watermelon": 19889, + "waterproof": 17613, + "waters": 7753, + "watershed": 33204, + "waterstones": 45014, + "waterways": 37395, + "watford": 23162, + "watfordfc": 37328, + "wati": 27966, + "watkins": 22539, + "watson": 35490, + "watson": 9294, + "watt": 22899, + "watt": 15805, + "wattpad": 32351, + "watts": 14750, + "wau": 9479, + "wav": 6054, + "wave": 17530, + "wave": 4535, + "waved": 44657, + "waver": 25997, + "waves": 7882, + "waving": 26545, + "wavy": 31941, + "waw": 22039, + "wawrinka": 48414, + "wawx": 47387, + "wax": 18789, + "wax": 11910, + "waxing": 38781, + "way": 3079, + "way": 923, + "wayback": 47822, + "wayne": 23632, + "wayne": 7003, + "ways": 1248, + "waz": 20889, + "waz": 48835, + "wb": 10726, + "wb": 12377, + "wba": 22675, + "wbb": 14482, + "wbc": 26745, + "wbo": 49053, + "wbz": 35471, + "wc": 4842, + "wc": 5755, + "wcc": 47166, + "wcc": 34926, + "wcpo": 46624, + "wcs": 39916, + "wcvb": 32709, + "wcw": 9041, + "wd": 15998, + "wd": 7494, + "wdw": 40334, + "we": 598, + "we": 649, + "wea": 37146, + "wea": 47301, + "weak": 12128, + "weak": 10128, + "weaker": 39735, + "weakness": 21448, + "weaknesses": 43487, + "weal": 14759, + "wealth": 33150, + "wealth": 7904, + "wealthy": 22617, + "weap": 6156, + "weapon": 42612, + "weapon": 10537, + "weapons": 10007, + "wear": 12206, + "wear": 2839, + "wearab": 22983, + "wearable": 44943, + "wearable": 24973, + "wearables": 30319, + "weare": 4264, + "weare": 27867, + "weareall": 45980, + "wearec": 43620, + "wearen": 45635, + "weareone": 16149, + "weareoneexo": 16448, + "wearethe": 40242, + "wearing": 3309, + "wears": 11869, + "weary": 38766, + "weasel": 44308, + "weather": 8808, + "weather": 2237, + "weathercee": 44980, + "weatherchannel": 42138, + "weav": 22260, + "weave": 22450, + "weaver": 20297, + "weaving": 27131, + "web": 2055, + "web": 4601, + "webb": 15708, + "webber": 34248, + "webcam": 24211, + "webcam": 22589, + "webcamtoy": 27719, + "webcast": 28256, + "webcomic": 34286, + "webcomics": 39811, + "webdesign": 20470, + "webdev": 37000, + "webdevelopment": 47553, + "weber": 20179, + "webin": 8460, + "webinar": 8921, + "webinars": 47755, + "webpage": 46964, + "webs": 32829, + "webseries": 44819, + "website": 3364, + "websites": 19278, + "webster": 19471, + "websummit": 48069, + "wec": 33152, + "wechat": 46124, + "wed": 1687, + "wed": 3478, + "wedd": 7576, + "wedding": 11204, + "wedding": 3101, + "weddings": 15964, + "wedge": 21446, + "wedges": 33179, + "wedne": 2380, + "wednesday": 9311, + "wednesday": 2689, + "wednesdaymotivation": 37860, + "wednesdays": 24943, + "wednesdaywisdom": 11445, + "wedo": 43432, + "weds": 19107, + "wee": 716, + "wee": 8288, + "weed": 36935, + "weed": 8015, + "weeds": 26326, + "week": 1286, + "week": 994, + "weekday": 29244, + "weekdays": 44330, + "weekend": 17205, + "weekend": 1456, + "weekender": 36547, + "weekends": 14564, + "weekly": 34652, + "weekly": 5885, + "weeknd": 29925, + "weeks": 2898, + "weeksary": 24628, + "ween": 17517, + "ween": 1599, + "weep": 39270, + "weeping": 36629, + "weer": 32491, + "weet": 17742, + "weets": 13454, + "wef": 23313, + "weg": 47867, + "weg": 47561, + "wego": 44784, + "wego": 28220, + "weh": 48458, + "weh": 40313, + "weho": 47798, + "wei": 6958, + "wei": 20952, + "weibo": 20613, + "weigh": 10565, + "weigh": 17346, + "weighed": 33210, + "weighing": 24455, + "weighs": 20481, + "weight": 12723, + "weight": 3868, + "weighted": 43179, + "weightlifting": 36164, + "weightloss": 20359, + "weights": 21374, + "weil": 43720, + "weiler": 42203, + "wein": 29134, + "wein": 37684, + "weiner": 38822, + "weinstein": 34367, + "weir": 11299, + "weir": 25517, + "weird": 27981, + "weird": 5613, + "weirdest": 29482, + "weirdo": 32476, + "weis": 26251, + "weiser": 34833, + "weiss": 24794, + "wel": 1267, + "wel": 8042, + "welch": 25820, + "welcom": 11578, + "welcome": 18318, + "welcome": 1881, + "welcomed": 12590, + "welcomes": 9304, + "welcometo": 47511, + "welcoming": 8775, + "weld": 39776, + "welding": 24956, + "welfare": 12129, + "well": 3277, + "well": 1123, + "wellbeing": 14273, + "weller": 40921, + "welling": 49165, + "wellington": 15389, + "wellness": 40574, + "wellness": 9904, + "wells": 42705, + "wells": 9804, + "welove": 13573, + "welp": 28391, + "wels": 20852, + "welsh": 19173, + "welsh": 10977, + "welt": 38595, + "welter": 37115, + "welterweight": 39617, + "wemb": 15213, + "wembley": 16579, + "wen": 6590, + "wen": 11278, + "wend": 15166, + "wendell": 42091, + "wendy": 31616, + "wendy": 14074, + "wenger": 21105, + "went": 18633, + "went": 2437, + "wentworth": 36423, + "wentz": 39179, + "wer": 6316, + "wer": 2980, + "were": 15461, + "were": 1365, + "wered": 6605, + "weren": 13611, + "werewolf": 32001, + "werk": 30176, + "werner": 29917, + "wers": 7110, + "wes": 18620, + "wes": 14738, + "wesle": 29606, + "wesley": 17332, + "wesleyan": 32509, + "wesome": 33292, + "wess": 44431, + "west": 2973, + "west": 1593, + "westbound": 29208, + "westbrook": 26948, + "westchester": 36675, + "westcoast": 44610, + "westend": 44815, + "wester": 9846, + "western": 17079, + "western": 4463, + "westfield": 32309, + "westh": 36798, + "westin": 43232, + "westlake": 41535, + "westminster": 15158, + "weston": 22771, + "westside": 33762, + "westwood": 26371, + "westworld": 42287, + "wet": 12406, + "wet": 6682, + "weta": 40946, + "wethenorth": 45281, + "wethepeople": 48030, + "wether": 33794, + "wether": 48405, + "wetland": 37357, + "wetlands": 26547, + "wett": 41971, + "wetter": 43957, + "wewant": 39280, + "wewill": 37241, + "wex": 17234, + "wexford": 29876, + "wexmondays": 49042, + "wey": 30376, + "wey": 19781, + "weymouth": 41433, + "wf": 14576, + "wf": 22313, + "wfa": 44606, + "wfc": 36431, + "wfp": 35193, + "wftv": 47075, + "wg": 21091, + "wg": 25857, + "wga": 32354, + "wgn": 48828, + "wh": 573, + "wh": 13844, + "wha": 18994, + "wha": 25884, + "whal": 38967, + "whale": 37083, + "whale": 11650, + "whales": 17722, + "wham": 42506, + "whar": 15517, + "wharf": 22452, + "wharton": 43320, + "what": 4268, + "what": 768, + "whatcha": 37160, + "whate": 6695, + "whatever": 6743, + "whati": 23500, + "whats": 9263, + "whats": 13084, + "whatsapp": 10119, + "whatsoever": 39928, + "whatson": 35632, + "whatyou": 30508, + "whe": 2009, + "whead": 34583, + "wheat": 20505, + "wheat": 10303, + "wheaton": 46933, + "wheel": 7360, + "wheel": 6744, + "wheelchair": 17713, + "wheeler": 18405, + "wheeling": 34839, + "wheels": 8025, + "whel": 9792, + "whelan": 40715, + "when": 8753, + "when": 827, + "whenever": 10500, + "where": 7052, + "where": 1234, + "whereabouts": 47808, + "whereas": 42234, + "wheres": 46345, + "wherever": 14103, + "whereyou": 46837, + "whether": 5903, + "whew": 39016, + "whey": 34556, + "whi": 4295, + "whi": 33129, + "which": 1448, + "whiche": 48719, + "whichever": 49138, + "whil": 8499, + "while": 1519, + "whilst": 8596, + "whim": 27766, + "whimsical": 42282, + "whip": 14412, + "whipped": 22323, + "whipping": 41567, + "whir": 20873, + "whirl": 30962, + "whirlwind": 47771, + "whis": 6024, + "whiskey": 41381, + "whiskey": 11610, + "whisky": 37567, + "whisky": 12599, + "whisp": 21986, + "whispe": 30356, + "whisper": 27616, + "whisperer": 41368, + "whispering": 42599, + "whispers": 29133, + "whist": 13640, + "whistle": 23972, + "whistle": 19746, + "whistleblower": 40410, + "whistler": 29633, + "whit": 4398, + "whit": 31498, + "whitaker": 35851, + "whitby": 30858, + "white": 4699, + "white": 1579, + "whiteboard": 40839, + "whitec": 24575, + "whitehall": 42827, + "whitehead": 43560, + "whitehouse": 20776, + "whitening": 35540, + "whitepaper": 42713, + "whites": 35886, + "whites": 18835, + "whitesox": 28816, + "whitewater": 49350, + "whitfield": 48404, + "whitley": 40564, + "whitman": 32394, + "whitney": 43021, + "whitney": 18048, + "whitt": 33784, + "whittaker": 47595, + "whl": 25801, + "who": 2969, + "who": 822, + "whoa": 16943, + "whoever": 11137, + "whois": 41884, + "whole": 10360, + "whole": 2954, + "wholefoods": 42840, + "wholesale": 18306, + "wholesome": 35959, + "whom": 38158, + "whom": 12873, + "whoo": 20003, + "whoo": 49290, + "whoop": 22060, + "whoops": 28433, + "whopping": 34384, + "whore": 31690, + "whos": 41460, + "whos": 27130, + "whose": 6933, + "whouse": 45927, + "whs": 26292, + "wht": 32470, + "whufc": 31695, + "whun": 18272, + "why": 11040, + "why": 1182, + "whyte": 42386, + "wi": 820, + "wi": 5585, + "wib": 45303, + "wic": 7834, + "wich": 9759, + "wich": 5238, + "wichita": 22566, + "wick": 6798, + "wick": 6479, + "wicked": 32579, + "wicked": 12825, + "wicker": 38096, + "wicket": 19180, + "wickets": 22110, + "wicklow": 39039, + "wicz": 30121, + "wid": 11886, + "wid": 20886, + "wide": 19341, + "wide": 3184, + "widely": 16195, + "widening": 46598, + "wider": 21263, + "widesp": 20598, + "widespread": 21258, + "widget": 43906, + "wido": 28068, + "widow": 19949, + "widows": 42129, + "width": 23571, + "wie": 21378, + "wie": 9131, + "wielding": 47272, + "wien": 38131, + "wiener": 40567, + "wies": 42788, + "wif": 37572, + "wife": 3607, + "wifey": 35282, + "wifi": 11026, + "wig": 23690, + "wig": 12216, + "wigan": 23130, + "wiggins": 32329, + "wiggle": 47812, + "wight": 41278, + "wight": 15545, + "wigs": 31207, + "wii": 8005, + "wiiu": 40980, + "wiki": 10373, + "wiki": 24265, + "wikileaks": 28731, + "wikipedia": 15176, + "wil": 1352, + "wil": 20581, + "wilbur": 43069, + "wilcox": 43231, + "wild": 2780, + "wild": 3220, + "wildatlantic": 35500, + "wildatlanticway": 35776, + "wildcard": 37360, + "wildcat": 49077, + "wildcat": 25870, + "wildcats": 15909, + "wilde": 23498, + "wilder": 14343, + "wilder": 23499, + "wilderness": 16506, + "wildest": 43028, + "wildfire": 22788, + "wildfires": 29184, + "wildflower": 27628, + "wildflower": 33181, + "wildflowerhour": 31302, + "wildflowers": 29136, + "wildlife": 13298, + "wildlife": 5250, + "wildlifephotography": 32307, + "wildlifewednesday": 48537, + "wildly": 35981, + "wildoz": 40113, + "wiley": 32747, + "wilhelm": 39696, + "wilkes": 39548, + "wilkins": 36986, + "wilkinson": 26797, + "will": 5062, + "will": 751, + "willam": 43276, + "willard": 44920, + "wille": 48739, + "willem": 38044, + "willi": 2256, + "william": 8420, + "william": 4705, + "williams": 38452, + "williams": 4075, + "williamsburg": 30683, + "williamson": 20793, + "willie": 13907, + "willing": 34160, + "willing": 11718, + "willingness": 40573, + "willis": 18491, + "willow": 33887, + "willow": 15665, + "wills": 26913, + "willy": 34502, + "willy": 19599, + "wilmington": 28052, + "wilms": 47879, + "wilshere": 48359, + "wilson": 23629, + "wilson": 5622, + "wilt": 23394, + "wilt": 47357, + "wilton": 46638, + "wiltshire": 28025, + "wim": 8662, + "wim": 27580, + "wimble": 11752, + "wimbledon": 12229, + "win": 831, + "win": 1225, + "winchester": 20647, + "wind": 6812, + "wind": 3630, + "winder": 44454, + "winder": 46245, + "winding": 22390, + "windmill": 34084, + "windo": 3110, + "window": 26675, + "window": 4879, + "windows": 5437, + "winds": 12668, + "winds": 7012, + "windshield": 33002, + "windsor": 44322, + "windsor": 12884, + "windy": 13446, + "wine": 7375, + "wine": 2604, + "winelover": 26357, + "winemaker": 41588, + "wineoclock": 43846, + "wineries": 49349, + "winery": 15500, + "wines": 8263, + "winetasting": 41288, + "winewednesday": 35447, + "wing": 8141, + "wing": 1340, + "winged": 24993, + "winger": 22727, + "winget": 44578, + "wings": 5178, + "wink": 34455, + "wink": 25859, + "winkle": 36430, + "winn": 38104, + "winne": 46273, + "winner": 32961, + "winner": 2520, + "winners": 4320, + "winni": 13018, + "winnie": 29022, + "winning": 42099, + "winning": 2577, + "winnings": 46490, + "winnipeg": 14369, + "winona": 49202, + "wins": 46839, + "wins": 2718, + "winslow": 39658, + "winston": 14848, + "winter": 7340, + "winter": 2541, + "winters": 21587, + "wintry": 39504, + "wip": 10447, + "wipe": 26761, + "wiped": 31822, + "wipes": 33463, + "wir": 16849, + "wir": 44838, + "wire": 7558, + "wire": 7794, + "wired": 18935, + "wireless": 9103, + "wires": 24311, + "wiring": 36434, + "wirral": 34675, + "wis": 3392, + "wis": 20405, + "wiscon": 9857, + "wisconsin": 10265, + "wisdom": 42474, + "wisdom": 5425, + "wise": 19116, + "wise": 5558, + "wisely": 26173, + "wiser": 44859, + "wish": 11328, + "wish": 2412, + "wished": 25883, + "wishes": 6045, + "wishing": 5307, + "wishlist": 31969, + "wit": 584, + "wit": 8531, + "witch": 20139, + "witch": 10083, + "witchcraft": 35065, + "witcher": 33684, + "witches": 21673, + "with": 1435, + "with": 593, + "withdra": 24696, + "withdraw": 31670, + "withdrawal": 25765, + "withdrawn": 46687, + "withdraws": 48637, + "wither": 39655, + "witherspoon": 45409, + "within": 4154, + "withme": 44670, + "without": 32836, + "without": 2193, + "withstand": 42236, + "withthe": 36872, + "withus": 30572, + "withyou": 30351, + "witne": 12096, + "witness": 8793, + "witnessed": 20187, + "witnesses": 22778, + "witnessing": 33618, + "wits": 30938, + "witt": 38194, + "witt": 17168, + "witter": 31597, + "witty": 29970, + "witz": 44186, + "witz": 13265, + "wiv": 48925, + "wives": 14378, + "wiwx": 44461, + "wiz": 7730, + "wiz": 23178, + "wizar": 49121, + "wizard": 30490, + "wizard": 14295, + "wizards": 19140, + "wizkid": 40146, + "wj": 19739, + "wj": 35453, + "wk": 11512, + "wk": 11528, + "wkend": 42336, + "wknd": 20851, + "wks": 25508, + "wku": 43377, + "wl": 13299, + "wl": 9613, + "wm": 20268, + "wm": 15790, + "wn": 1186, + "wn": 757, + "wnba": 32358, + "wned": 8628, + "wns": 12950, + "wnt": 22484, + "wny": 24833, + "wo": 1613, + "wo": 11132, + "woah": 17751, + "wob": 35984, + "woc": 39011, + "wod": 41522, + "woes": 27860, + "wof": 45671, + "woj": 48931, + "wok": 28912, + "woke": 9331, + "woken": 43697, + "woking": 43931, + "wol": 2798, + "wol": 48622, + "wold": 42399, + "wolf": 9453, + "wolf": 5916, + "wolfe": 24989, + "wolff": 34369, + "wolfgang": 34061, + "wolfpack": 30887, + "wolve": 45101, + "wolver": 14334, + "wolverhampton": 34518, + "wolverine": 23353, + "wolverines": 42003, + "wolves": 9372, + "wom": 1087, + "womack": 48980, + "woman": 15716, + "woman": 2308, + "womanc": 35630, + "womancrush": 37721, + "womancrushwednesday": 39714, + "womanin": 30562, + "womaninbiz": 36482, + "womb": 37023, + "women": 3648, + "women": 1507, + "womenin": 13062, + "womeninscience": 41343, + "womeninstem": 29380, + "womenintech": 31470, + "womenof": 48421, + "womens": 12822, + "womens": 14408, + "womensart": 38548, + "womensday": 13956, + "womenshi": 22887, + "womenshistorymonth": 24982, + "womensmarch": 30102, + "won": 1528, + "won": 1749, + "wonder": 2070, + "wonder": 3936, + "wondercon": 46944, + "wondered": 15550, + "wonderful": 2582, + "wonderfully": 23245, + "wondering": 8360, + "wonderland": 13874, + "wonders": 14048, + "wonderwoman": 31000, + "wondo": 38402, + "wondr": 46771, + "wong": 17876, + "wonka": 43463, + "wont": 43174, + "wont": 15952, + "woo": 1867, + "woo": 9322, + "wood": 3269, + "wood": 1704, + "woodbridge": 49074, + "wooden": 48226, + "wooden": 9057, + "woodland": 44314, + "woodland": 17447, + "woodlands": 32430, + "woodley": 40566, + "woodpecker": 32684, + "woods": 6267, + "woodson": 48967, + "woodstock": 29486, + "woodward": 27419, + "woodwork": 47386, + "woodworking": 29267, + "woody": 38627, + "woody": 17144, + "woof": 34234, + "woof": 24028, + "woohoo": 20172, + "wook": 29192, + "wool": 9967, + "wool": 13283, + "woolf": 43728, + "woolly": 47722, + "woon": 33126, + "wooo": 43217, + "woop": 31884, + "woot": 22466, + "wor": 641, + "worcester": 22172, + "worcester": 19580, + "worcestershire": 38440, + "worcestershirehour": 43644, + "word": 8272, + "word": 2653, + "wordof": 33500, + "wordoftheday": 43594, + "wordpress": 15193, + "words": 31007, + "words": 2709, + "wore": 8953, + "work": 1636, + "work": 951, + "workday": 29735, + "worked": 5410, + "worker": 8098, + "workers": 4795, + "workflow": 28502, + "workforce": 14672, + "workin": 31825, + "workin": 26323, + "working": 20806, + "working": 1699, + "workinprogress": 46086, + "workout": 6773, + "workouts": 22779, + "workplace": 11959, + "workplaces": 47383, + "works": 2322, + "workshop": 3832, + "workshops": 12262, + "workspace": 34470, + "worl": 5221, + "world": 2334, + "world": 1002, + "worlda": 46627, + "worldbank": 36759, + "worldbookday": 31191, + "worldcup": 42525, + "worldcup": 8650, + "worlden": 44668, + "worldenviron": 47115, + "worldenvironmentday": 47522, + "worldly": 36268, + "worldo": 41698, + "worldof": 22636, + "worldre": 33951, + "worlds": 7691, + "worldseries": 26695, + "worldtour": 23202, + "worldwater": 41176, + "worldwaterday": 44520, + "worldwide": 6214, + "worm": 33709, + "worm": 10945, + "worms": 20231, + "worn": 9037, + "worried": 11911, + "worries": 17684, + "worry": 7534, + "worrying": 24058, + "worse": 8236, + "worsen": 46344, + "worshi": 31840, + "worship": 46399, + "worship": 9023, + "worst": 5719, + "wort": 30209, + "worth": 10671, + "worth": 2450, + "worthing": 39929, + "worthit": 40830, + "worthless": 44736, + "worths": 44633, + "worthwhile": 36295, + "worthy": 8881, + "worx": 44973, + "wot": 24863, + "wou": 5279, + "would": 39873, + "would": 1311, + "wouldn": 5878, + "wouldnt": 41595, + "wound": 19231, + "wounded": 14859, + "wounds": 21290, + "woven": 19830, + "wow": 22191, + "wow": 2781, + "woz": 44558, + "wozni": 47782, + "wp": 15378, + "wp": 13302, + "wpg": 35048, + "wps": 33386, + "wq": 45195, + "wr": 1189, + "wr": 8028, + "wra": 3852, + "wra": 46004, + "wral": 49050, + "wrangler": 30923, + "wrap": 7094, + "wrapped": 9875, + "wrapping": 15223, + "wraps": 18236, + "wrath": 29783, + "wray": 48943, + "wrc": 16004, + "wre": 3168, + "wreath": 23091, + "wrec": 20879, + "wreck": 28775, + "wreck": 15017, + "wrecked": 32695, + "wreckem": 45676, + "wrecking": 36956, + "wrecks": 45545, + "wren": 20191, + "wren": 31970, + "wrench": 30980, + "wrest": 4177, + "wrestle": 17097, + "wrestle": 28086, + "wrestlemania": 18849, + "wrestler": 19790, + "wrestlers": 25902, + "wrestling": 31292, + "wrestling": 5904, + "wrexham": 34479, + "wri": 7667, + "wri": 42007, + "wright": 28616, + "wright": 6991, + "wrights": 43711, + "wrigley": 33538, + "wrink": 22201, + "wrinkle": 46642, + "wrinkles": 35525, + "wrist": 19243, + "wrist": 16139, + "wristband": 36890, + "wristbands": 44864, + "writ": 2902, + "write": 28874, + "write": 4946, + "writer": 27886, + "writer": 4422, + "writers": 18742, + "writers": 7307, + "writerslife": 25007, + "writes": 8023, + "writing": 16053, + "writing": 2979, + "writingcommunity": 39178, + "writings": 36259, + "written": 5231, + "wro": 5447, + "wrong": 18381, + "wrong": 3669, + "wrongly": 45642, + "wrote": 5796, + "wrought": 48125, + "wrs": 45280, + "ws": 6300, + "ws": 799, + "wsb": 30681, + "wsbtv": 38394, + "wsj": 19764, + "wski": 12548, + "wsl": 43706, + "wsoc": 40253, + "wson": 33954, + "wsop": 41231, + "wsu": 44674, + "wsu": 32913, + "wsw": 43285, + "wt": 15873, + "wt": 12255, + "wta": 25984, + "wtc": 39718, + "wtf": 6891, + "wth": 23021, + "wthr": 45269, + "wti": 47345, + "wto": 36406, + "wts": 32159, + "wu": 9710, + "wu": 9837, + "wud": 43870, + "wul": 35154, + "wunder": 36661, + "wur": 24040, + "wurst": 44409, + "wusa": 40021, + "wut": 28590, + "wv": 18920, + "wv": 14743, + "wvu": 44878, + "wvu": 25879, + "ww": 3181, + "ww": 4491, + "wwc": 26505, + "wwdc": 47441, + "wwe": 12112, + "wwe": 5290, + "wwen": 23308, + "wwenetwork": 37228, + "wwenxt": 39898, + "wwer": 32038, + "wwf": 23332, + "wwfc": 42681, + "wwg": 35322, + "wwi": 20194, + "wwii": 10261, + "www": 26074, + "www": 9667, + "wwwbigbaldhead": 30761, + "wwww": 34224, + "wwww": 25200, + "wwwww": 48268, + "wwx": 47431, + "wx": 18192, + "wx": 3561, + "wy": 4665, + "wy": 7625, + "wyatt": 21660, + "wyd": 33113, + "wye": 48436, + "wye": 43751, + "wylie": 49330, + "wyn": 11802, + "wyn": 17504, + "wynn": 36117, + "wynne": 35951, + "wynonna": 41456, + "wynonnaearp": 43755, + "wyoming": 18693, + "x": 87, + "x": 343, + "xa": 24831, + "xan": 45530, + "xander": 45601, + "xavi": 36342, + "xavier": 41044, + "xavier": 18567, + "xb": 33678, + "xbox": 18063, + "xbox": 7748, + "xboxone": 27410, + "xc": 12515, + "xchange": 49132, + "xd": 6380, + "xe": 42886, + "xe": 19183, + "xen": 15568, + "xer": 49005, + "xf": 35274, + "xfactor": 25211, + "xfinity": 35107, + "xford": 34732, + "xh": 45771, + "xham": 25284, + "xi": 2467, + "xi": 7376, + "xia": 19854, + "xia": 20724, + "xian": 42570, + "xiao": 49318, + "xiaomi": 27477, + "xico": 38469, + "xide": 17398, + "xie": 40122, + "xie": 15976, + "xii": 36525, + "xiii": 28199, + "xim": 11217, + "xin": 27053, + "xin": 41517, + "xing": 14383, + "xion": 24164, + "xis": 35793, + "xit": 5316, + "xiumin": 36563, + "xiv": 16125, + "xj": 42453, + "xl": 36529, + "xl": 8833, + "xley": 38223, + "xm": 18626, + "xma": 48805, + "xmas": 48848, + "xmas": 6425, + "xmen": 28708, + "xn": 25388, + "xo": 26936, + "xo": 9000, + "xon": 29186, + "xon": 8482, + "xox": 11531, + "xox": 34050, + "xoxo": 13313, + "xp": 15651, + "xper": 32200, + "xperia": 37615, + "xpo": 44377, + "xpress": 31809, + "xq": 40606, + "xr": 26276, + "xrp": 26965, + "xs": 16397, + "xt": 1052, + "xtina": 45520, + "xton": 32666, + "xton": 10597, + "xtra": 26969, + "xtre": 27025, + "xtreme": 33483, + "xu": 42063, + "xu": 37198, + "xv": 17768, + "xvi": 44031, + "xx": 5675, + "xx": 3553, + "xxl": 29777, + "xxx": 33923, + "xxx": 8352, + "xxxx": 32035, + "xxxx": 22819, + "xxxxx": 44195, + "xy": 20023, + "xy": 11443, + "y": 88, + "y": 344, + "ya": 5018, + "ya": 1430, + "yaa": 48847, + "yaa": 34498, + "yaan": 34680, + "yab": 27737, + "yach": 9039, + "yacht": 43806, + "yacht": 12859, + "yachts": 29260, + "yad": 13276, + "yad": 40047, + "yadav": 26650, + "yaf": 38019, + "yag": 35081, + "yah": 16170, + "yah": 12381, + "yaho": 37929, + "yahoo": 38152, + "yahoo": 16846, + "yak": 11014, + "yak": 29074, + "yaki": 44677, + "yaku": 29572, + "yakuza": 42628, + "yal": 16198, + "yal": 13418, + "yale": 39926, + "yale": 17157, + "yall": 9210, + "yam": 6666, + "yam": 19318, + "yama": 23512, + "yamaha": 18854, + "yan": 3949, + "yan": 4788, + "yana": 18698, + "yand": 38609, + "yang": 23818, + "yang": 12605, + "yani": 26439, + "yankee": 21554, + "yankees": 11889, + "yann": 40246, + "yann": 38657, + "yao": 45231, + "yap": 48700, + "yap": 34468, + "yar": 6786, + "yar": 23071, + "yard": 20234, + "yard": 4313, + "yards": 7550, + "yarmouth": 45941, + "yarn": 19702, + "yarra": 46824, + "yas": 8168, + "yas": 20570, + "yash": 30216, + "yash": 37836, + "yasi": 37700, + "yasss": 23873, + "yat": 29443, + "yat": 34965, + "yates": 27677, + "yatra": 38932, + "yav": 41275, + "yaw": 31989, + "yawn": 48643, + "yay": 20614, + "yay": 6712, + "yaya": 37608, + "yaz": 19348, + "yaz": 42252, + "yb": 41785, + "yb": 27615, + "yc": 11931, + "ycle": 38089, + "yd": 29896, + "yd": 9534, + "yday": 15899, + "yds": 24819, + "ye": 693, + "ye": 4582, + "yea": 13687, + "yeah": 29405, + "yeah": 3908, + "year": 5163, + "year": 935, + "yearbook": 21636, + "yearling": 48392, + "yearly": 24541, + "yearof": 31944, + "yearofthe": 47899, + "years": 30864, + "years": 1151, + "yearsof": 14932, + "yearswith": 45249, + "yeast": 25819, + "yeats": 44903, + "yed": 28137, + "yed": 3301, + "yee": 18114, + "yee": 23108, + "yeezy": 24901, + "yeg": 16854, + "yeg": 11976, + "yegfood": 48711, + "yeh": 21331, + "yel": 3323, + "yel": 48164, + "yell": 30824, + "yelled": 39199, + "yelling": 26581, + "yellow": 12059, + "yellow": 4481, + "yellowstone": 29241, + "yelp": 31674, + "yemen": 29276, + "yemen": 12513, + "yemeni": 44656, + "yemi": 42267, + "yen": 29602, + "yen": 17960, + "yeo": 32292, + "yeo": 43830, + "yeol": 15808, + "yeon": 16602, + "yep": 10964, + "yer": 15491, + "yer": 2371, + "yers": 3722, + "yes": 21620, + "yes": 1958, + "yess": 42778, + "yess": 40189, + "yesss": 36210, + "yessss": 45620, + "yester": 1905, + "yesterday": 1926, + "yesterdays": 36238, + "yesung": 38527, + "yet": 2296, + "yeti": 34228, + "yev": 39855, + "yew": 34660, + "yey": 45447, + "yg": 16396, + "ygk": 44758, + "ygo": 46166, + "yh": 41978, + "yi": 5826, + "yi": 14762, + "yield": 16825, + "yields": 24856, + "yikes": 25094, + "yin": 26476, + "yin": 23543, + "ying": 42933, + "ying": 910, + "yixing": 32120, + "yk": 30965, + "yl": 2656, + "yl": 4045, + "ylan": 41875, + "ylde": 42850, + "yle": 32305, + "yle": 10770, + "ylene": 34239, + "yler": 48081, + "yles": 42860, + "ylon": 22375, + "ylor": 48468, + "ym": 1786, + "ym": 19587, + "yman": 29077, + "ymc": 47101, + "ymca": 22369, + "yment": 8199, + "ymes": 39968, + "ymi": 5271, + "ymm": 37133, + "ymoun": 41426, + "ymouth": 36429, + "yn": 2823, + "yn": 4100, + "yne": 18238, + "ynes": 18020, + "ynn": 10499, + "ynna": 48292, + "ynwa": 27372, + "yo": 586, + "yo": 3497, + "yoda": 31922, + "yof": 5966, + "yofficial": 21818, + "yofthe": 43983, + "yog": 34985, + "yog": 36539, + "yoga": 25872, + "yoga": 5523, + "yogh": 32626, + "yoghurt": 33491, + "yogi": 22766, + "yogur": 16137, + "yogurt": 16819, + "yoh": 48880, + "yoke": 41969, + "yoko": 25929, + "yoko": 32256, + "yokohama": 42409, + "yol": 19387, + "yol": 35218, + "yolanda": 43845, + "yolo": 20905, + "yom": 34718, + "yom": 44527, + "yon": 10147, + "yon": 7604, + "yong": 27960, + "yong": 20887, + "yonge": 48592, + "yoo": 25842, + "yoo": 20775, + "yoon": 30863, + "yoon": 22113, + "yoona": 32736, + "yoongi": 24037, + "yor": 2028, + "yor": 21132, + "york": 5318, + "york": 2705, + "yorker": 23865, + "yorkers": 41041, + "yorks": 39093, + "yorkshi": 43367, + "yorkshire": 27007, + "yorkshire": 8633, + "yoruba": 46083, + "yos": 35607, + "yosemite": 25893, + "yoshi": 22920, + "yoshi": 25354, + "yot": 22875, + "yotes": 46157, + "yotpo": 26113, + "you": 1562, + "you": 592, + "youare": 33879, + "youcan": 32498, + "youknow": 47919, + "youknow": 41088, + "youn": 1596, + "young": 6939, + "young": 1888, + "younger": 10414, + "youngest": 12316, + "youngjae": 46426, + "youngster": 35881, + "youngsters": 28098, + "younow": 33831, + "your": 2130, + "your": 695, + "youre": 28344, + "youre": 19695, + "yourown": 28583, + "yours": 3834, + "yourself": 3053, + "yourselves": 19747, + "youth": 10743, + "youth": 3281, + "youthful": 37480, + "youths": 23614, + "youts": 22737, + "youtu": 13868, + "youtube": 31258, + "youtube": 3895, + "youtuber": 24720, + "youtubers": 36822, + "youu": 35055, + "youuu": 35324, + "youuuu": 47123, + "yoy": 41865, + "yp": 38370, + "yp": 34734, + "ypg": 37386, + "yql": 46122, + "yqr": 36881, + "yr": 18395, + "yr": 4333, + "yrs": 4822, + "ys": 1971, + "ys": 961, + "yser": 33121, + "ysis": 4843, + "ysl": 45681, + "ysm": 23842, + "yst": 40528, + "yt": 36777, + "yt": 14779, + "ytd": 47524, + "yte": 48172, + "yu": 3371, + "yu": 8887, + "yuan": 26236, + "yuck": 48282, + "yugo": 48231, + "yuh": 42547, + "yui": 47932, + "yuk": 17037, + "yuk": 24063, + "yuki": 34010, + "yukon": 27094, + "yul": 39832, + "yum": 6869, + "yum": 7259, + "yuma": 47566, + "yummy": 7687, + "yun": 14976, + "yun": 18288, + "yung": 44545, + "yung": 17676, + "yunho": 39748, + "yup": 13231, + "yur": 42533, + "yuri": 23823, + "yusuf": 33222, + "yuv": 36784, + "yves": 33698, + "yvon": 23327, + "yvonne": 32583, + "yvr": 29058, + "yw": 33741, + "yx": 35624, + "yxe": 34240, + "yy": 3433, + "yy": 8321, + "yya": 37444, + "yyc": 27542, + "yyc": 11741, + "yyj": 26203, + "yyy": 11514, + "yyyy": 38749, + "yyyy": 16955, + "yyyyy": 26089, + "yyyyyy": 47055, + "yz": 37579, + "yz": 46451, + "yü": 48232, + "z": 89, + "z": 345, + "za": 3710, + "za": 2186, + "zab": 22982, + "zable": 37002, + "zac": 25501, + "zac": 19159, + "zach": 13401, + "zach": 11815, + "zachary": 32401, + "zack": 30567, + "zack": 19120, + "zad": 47314, + "zad": 27838, + "zada": 34889, + "zaf": 21837, + "zafar": 46668, + "zag": 26091, + "zag": 29346, + "zagre": 34107, + "zagreb": 35355, + "zah": 23258, + "zah": 43297, + "zaha": 44408, + "zai": 44329, + "zai": 27065, + "zain": 34400, + "zain": 45366, + "zak": 13050, + "zak": 20738, + "zaki": 48091, + "zal": 20552, + "zal": 33298, + "zam": 7218, + "zam": 41578, + "zambia": 21671, + "zan": 7284, + "zan": 17835, + "zana": 39643, + "zand": 37712, + "zane": 34786, + "zani": 45373, + "zania": 15059, + "zano": 27637, + "zanzi": 47835, + "zap": 24134, + "zapp": 33504, + "zappa": 46592, + "zar": 5458, + "zar": 16392, + "zara": 24454, + "zardari": 20174, + "zas": 48261, + "zation": 3683, + "zawa": 49281, + "zay": 7102, + "zayed": 36726, + "zayn": 22292, + "zayn": 10308, + "zaynmalik": 25278, + "zazzle": 47857, + "ze": 2254, + "ze": 1298, + "zeal": 44951, + "zealand": 7618, + "zeb": 46518, + "zebra": 47394, + "zebra": 22548, + "zed": 21047, + "zed": 1993, + "zedd": 45608, + "zee": 25468, + "zee": 14080, + "zeiss": 47460, + "zeit": 37898, + "zeit": 37906, + "zek": 40829, + "zeke": 47065, + "zel": 10389, + "zel": 12027, + "zelda": 17138, + "zell": 39526, + "zen": 8518, + "zen": 3928, + "zend": 33478, + "zendaya": 35956, + "zenith": 44740, + "zens": 15298, + "zeph": 40726, + "zepp": 22977, + "zeppelin": 25408, + "zer": 6118, + "zer": 3716, + "zero": 14867, + "zero": 5848, + "zers": 9547, + "zes": 4073, + "zest": 37709, + "zet": 34098, + "zeta": 30954, + "zetta": 45993, + "zeus": 32800, + "zey": 46647, + "zh": 33389, + "zh": 41621, + "zhang": 21127, + "zhen": 37374, + "zhen": 33236, + "zhou": 17384, + "zhu": 42049, + "zi": 2651, + "zi": 5819, + "zia": 13764, + "zid": 30235, + "zidane": 34643, + "zie": 29316, + "zie": 8956, + "zieg": 40157, + "ziegler": 46812, + "ziel": 32151, + "zier": 15399, + "zies": 38001, + "ziest": 28159, + "zig": 15950, + "zig": 21345, + "ziggy": 39274, + "zik": 30125, + "zika": 28783, + "zil": 25039, + "zil": 33190, + "zilla": 17879, + "zim": 8112, + "zim": 22577, + "zimbab": 12373, + "zimbabwe": 45668, + "zimbabwe": 13583, + "zimmer": 27452, + "zimmer": 35211, + "zimmerman": 38231, + "zin": 14085, + "zin": 21278, + "zinc": 27458, + "zind": 26206, + "zindabad": 42208, + "zine": 16100, + "zing": 25062, + "zing": 3152, + "zinger": 42027, + "zio": 13906, + "zion": 31763, + "zion": 20963, + "zione": 36161, + "zionist": 33078, + "zip": 26479, + "zip": 16083, + "zipper": 33670, + "zir": 31892, + "zl": 39168, + "zlat": 32489, + "zlatan": 37877, + "zm": 43691, + "zman": 24248, + "zn": 18004, + "zo": 4397, + "zo": 5056, + "zodi": 22660, + "zodiac": 27753, + "zoe": 43114, + "zoe": 16662, + "zoey": 39871, + "zog": 40680, + "zol": 25939, + "zola": 46105, + "zom": 6623, + "zombi": 29452, + "zombie": 11819, + "zombies": 46702, + "zombies": 16517, + "zon": 15109, + "zon": 14618, + "zona": 42134, + "zone": 37197, + "zone": 4442, + "zones": 17247, + "zoning": 36790, + "zoo": 8182, + "zoo": 7147, + "zoom": 32671, + "zoom": 13909, + "zor": 17605, + "zou": 38072, + "zr": 39275, + "zs": 35248, + "zshq": 41442, + "zt": 42629, + "zu": 4091, + "zu": 14184, + "zucchini": 29873, + "zucker": 26890, + "zuckerberg": 30066, + "zul": 31146, + "zulu": 32821, + "zum": 35094, + "zuma": 23326, + "zumba": 32976, + "zun": 42440, + "zur": 17128, + "zurich": 21288, + "zw": 42188, + "zx": 31604, + "zy": 6615, + "zy": 2303, + "zyk": 39112, + "zyme": 36472, + "zyn": 45287, + "zz": 1544, + "zz": 4943, + "zza": 14642, + "zzi": 13974, + "zzie": 18635, + "zzle": 7873, + "zzled": 39075, + "zzo": 14036, + "zzy": 21275, + "zzy": 8353, + "zzz": 20055, + "zzzz": 35742, + "zzzz": 43103, + "{": 90, + "{": 346, + "{}": 39025, + "|": 91, + "|#": 31183, + "|": 347, + "|@": 41677, + "||": 7566, + "}": 92, + "}": 348, + "~": 93, + "~!": 31181, + "~\"": 48442, + "~": 349, + "~>": 43291, + "~@": 44247, + "~~": 11461, + "~~": 16671, + "~~~": 32472, + "~~~~": 28295, + "¡": 94, + "¡": 350, + "¡ï¸ı": 15113, + "¡ï¸ı": 4174, + "¡ľ": 43991, + "¢": 95, + "¢": 351, + "£": 96, + "£": 352, + "£ï¸ı": 18446, + "¤": 97, + "¤": 353, + "¥": 98, + "¥": 354, + "¦": 99, + "¦": 355, + "¦Ī": 47615, + "§": 100, + "§": 356, + "¨": 101, + "¨": 357, + "©": 102, + "©": 358, + "ª": 103, + "ª": 359, + "«": 104, + "«": 360, + "¬": 105, + "¬": 361, + "¬ë": 31736, + "®": 106, + "®": 362, + "¯": 107, + "¯": 363, + "°": 108, + "°:": 21787, + "°": 364, + "°ï¸ı": 34777, + "±": 109, + "±": 365, + "±ï¸ı": 41020, + "²": 110, + "²": 366, + "³": 111, + "³": 367, + "³ï¸ı": 22195, + "³ï¸ı": 24706, + "´": 112, + "´": 368, + "µ": 113, + "µ": 369, + "µï¸ı": 27605, + "¶": 114, + "¶": 370, + "·": 115, + "·": 371, + "¸": 116, + "¸": 372, + "¸ë": 19693, + "¹": 117, + "¹": 373, + "º": 118, + "º": 374, + "»": 119, + "»": 375, + "¼": 120, + "¼": 376, + "½": 121, + "½": 377, + "½ï¸ı": 31333, + "¾": 122, + "¾": 378, + "¿": 123, + "¿": 379, + "À": 124, + "À": 380, + "Á": 125, + "Á": 381, + "Â": 126, + "Â": 382, + "¡": 26868, + "¡": 10830, + "¡¡": 45505, + "¢": 41359, + "£": 31117, + "£": 1950, + "Â¥": 20199, + "¨": 19957, + "¨¨": 23089, + "¨¨¨¨": 41223, + "©": 31148, + "©": 5811, + "«": 14434, + "®": 30857, + "®": 8436, + "¯": 38682, + "¯": 43593, + "¯\\": 44096, + "¯\\_(": 45115, + "°": 21305, + "°": 6858, + "²": 41175, + "´": 30560, + "´": 12559, + "·": 14844, + "º": 28059, + "»": 31642, + "»": 7599, + "½": 33613, + "¿": 44559, + "¿": 17133, + "ÂŃ": 22618, + "Ã": 127, + "Ã": 383, + "á": 7261, + "á": 22229, + "án": 38340, + "án": 21385, + "â": 26170, + "ã": 19339, + "ão": 21141, + "ä": 10896, + "ä": 47276, + "än": 42787, + "Ã¥": 23176, + "æ": 42495, + "ç": 10067, + "ça": 22711, + "è": 12138, + "è": 37761, + "ère": 30272, + "ès": 41210, + "é": 3459, + "é": 4166, + "éal": 45251, + "ée": 13489, + "és": 20507, + "ê": 27515, + "ë": 29526, + "ë": 40520, + "î": 48704, + "ï": 35689, + "ñ": 6445, + "ña": 17753, + "ño": 16574, + "ños": 40104, + "ó": 8891, + "ó": 27733, + "ón": 13926, + "ô": 26815, + "ö": 7255, + "ö": 37423, + "ör": 31762, + "ø": 17483, + "ø": 45598, + "ú": 17963, + "ú": 36019, + "ü": 6522, + "ü": 47177, + "ür": 26132, + "ÃĹ": 16165, + "Ãł": 36149, + "Ãł": 21259, + "ÃŃ": 8366, + "ÃŃ": 23928, + "ÃŃa": 16609, + "ÃŃn": 33623, + "Ä": 128, + "Ä": 384, + "ı": 18562, + "ı": 41901, + "Äģ": 23134, + "Äĩ": 31719, + "Äį": 45414, + "ÄŁ": 26540, + "Å": 129, + "Å": 385, + "Å¡": 35621, + "ÅĤ": 40419, + "Åį": 41267, + "ÅŁ": 21254, + "ÅŁ": 40706, + "Æ": 130, + "Æ": 386, + "Ç": 131, + "Ç": 387, + "È": 132, + "È": 388, + "É": 133, + "É": 389, + "Ê": 134, + "Ê": 390, + "Ë": 135, + "Ë": 391, + "Ì": 136, + "Ì": 392, + "Ìĩ": 16384, + "Í": 137, + "Í": 393, + "Î": 138, + "Î": 394, + "Ï": 139, + "Ï": 395, + "Ïī": 38065, + "Ð": 140, + "Ð": 396, + "а": 16912, + "а": 27080, + "аÐ": 31090, + "в": 39813, + "е": 22176, + "и": 16701, + "иÐ": 29503, + "к": 27152, + "л": 47611, + "м": 38018, + "н": 22705, + "о": 13506, + "о": 29386, + "оÐ": 20978, + "од": 38416, + "оÑĤ": 28599, + "п": 26302, + "пÑĢи": 46321, + "пÑĢиÑĢода": 48150, + "Ñ": 141, + "Ñ": 397, + "ÑĢ": 16370, + "ÑĢи": 41092, + "ÑĢод": 47039, + "ÑĢода": 47929, + "Ñģ": 23669, + "ÑĤ": 17875, + "Ñĥ": 39729, + "ÑĦ": 27993, + "ÑĦоÑĤ": 35155, + "ÑĦоÑĤо": 38981, + "Ñĭ": 45001, + "Ò": 142, + "Ò": 398, + "Ó": 143, + "Ó": 399, + "Ô": 144, + "Ô": 400, + "Õ": 145, + "Õ": 401, + "Ö": 146, + "Ö": 402, + "×": 147, + "×": 403, + "Ø": 148, + "Ø": 404, + "ا": 6042, + "ا": 22625, + "اØ": 13189, + "ار": 40137, + "اÙ": 8453, + "اÙĦ": 12973, + "اÙħ": 47626, + "اÙĨ": 42773, + "اÙĨ": 33200, + "ب": 16378, + "ب": 35330, + "Ø©": 20915, + "ت": 18197, + "ت": 44333, + "ج": 26375, + "Ø®": 41495, + "د": 19872, + "د": 35566, + "ر": 10948, + "ر": 24933, + "رÙĬ": 43273, + "ز": 36169, + "س": 17856, + "Ø´": 28770, + "ص": 27271, + "Ø·": 32050, + "ع": 18843, + "غ": 48510, + "ØŃ": 25722, + "Ù": 149, + "Ù": 405, + "Ùģ": 24112, + "ÙĤ": 27585, + "Ùĥ": 33499, + "ÙĦ": 14251, + "ÙĦ": 37899, + "Ùħ": 12986, + "Ùħ": 29945, + "ÙĨ": 16655, + "ÙĨ": 25386, + "Ùĩ": 34274, + "Ùĩ": 31343, + "ÙĪ": 12203, + "ÙĪ": 38310, + "ÙĪر": 48242, + "ÙĬ": 12046, + "ÙĬ": 23853, + "Ú": 150, + "Ú": 406, + "Ú©": 26475, + "Û": 151, + "Û": 407, + "Ûģ": 40480, + "ÛĮ": 21452, + "ÛĮ": 32703, + "Ü": 152, + "Ü": 408, + "Ý": 153, + "Ý": 409, + "Þ": 154, + "Þ": 410, + "ß": 155, + "ß": 411, + "à": 156, + "à": 412, + "à¤": 3124, + "त": 27263, + "द": 29552, + "न": 26090, + "प": 44149, + "ब": 43599, + "म": 48254, + "म": 26774, + "य": 37299, + "र": 39136, + "र": 19052, + "ल": 30881, + "व": 39545, + "श": 43181, + "स": 28505, + "ह": 29446, + "ा": 37973, + "ा": 13343, + "ि": 26721, + "à¤Ĥ": 30833, + "à¤ķ": 22067, + "à¤Ĺ": 42598, + "à¤ľ": 39561, + "à¥": 7410, + "à¥Ģ": 45791, + "à¥Ģ": 25751, + "à¥ģ": 39653, + "à¥ĩ": 48612, + "à¥ĩ": 25130, + "à¥ĭ": 34452, + "à¥į": 19389, + "à¦": 11322, + "া": 41532, + "à§": 26339, + "à¨": 15741, + "à©": 32086, + "àª": 22990, + "à«": 48347, + "à¬": 32791, + "à®": 6022, + "த": 34691, + "ன": 43394, + "ப": 47388, + "à®®": 35463, + "à®°": 43270, + "ல": 47705, + "ா": 32831, + "ி": 27126, + "à®ķ": 36168, + "à®Ł": 45263, + "à¯": 11259, + "à¯ģ": 33115, + "à¯į": 16631, + "à°": 12100, + "à±": 23550, + "à±į": 46098, + "à²": 9992, + "ಿ": 47797, + "à³": 20745, + "à³į": 36148, + "à´": 15418, + "àµ": 27392, + "àµį": 45266, + "à¶": 29881, + "à·": 30766, + "à¸": 1777, + "ม": 26137, + "ม": 29570, + "ย": 27241, + "ย": 33091, + "ร": 32225, + "ร": 27331, + "ล": 34696, + "ล": 32746, + "ว": 26990, + "ว": 30245, + "ส": 37883, + "ส": 35737, + "ห": 33064, + "ะ": 43920, + "ะ": 49234, + "ั": 14978, + "า": 11529, + "า": 38476, + "าà¸": 12330, + "ิ": 17092, + "ี": 22421, + "ี": 20278, + "ีà¹Ī": 31511, + "ื": 47991, + "ุ": 30524, + "ู": 35273, + "à¸ģ": 30767, + "à¸ģà¸": 31474, + "à¸Ħ": 31757, + "à¸Ħà¸": 39628, + "à¸ĩ": 24603, + "à¸ĩ": 33382, + "à¸Ī": 47608, + "à¸Ĭ": 46324, + "à¸Ķ": 31107, + "à¸Ķ": 38825, + "à¸ķ": 40273, + "à¸ķ": 41108, + "à¸Ĺ": 36171, + "à¸Ļ": 17474, + "à¸Ļ": 17639, + "à¸Ļà¸": 23121, + "à¸ļ": 33859, + "à¸ļ": 39616, + "à¸ŀ": 48171, + "à¸Ń": 13398, + "à¸Ń": 32818, + "à¸Ńà¸": 14649, + "à¸Ńà¸ĩ": 46622, + "à¹": 4484, + "à¹Ģ": 13729, + "à¹Ģà¸": 14076, + "à¹ģà¸": 23916, + "à¹Ĥ": 33118, + "à¹ĥ": 40962, + "à¹Ħà¸": 31718, + "à¹ĩ": 38699, + "à¹Ī": 11722, + "à¹ī": 13123, + "à¹Į": 28353, + "à¼": 46186, + "à½": 39219, + "á": 157, + "á": 413, + "á´": 19036, + "áµ": 17330, + "áĢ": 45932, + "áĥ": 24829, + "áĥ¦": 32193, + "â": 158, + "â": 414, + "â¤": 25087, + "⤵ï¸ı": 36026, + "â¬": 7930, + "â¬ħï¸ı": 42111, + "â¬Ĩ": 27718, + "â¬Ĩï¸ı": 32798, + "â¬ĩ": 10917, + "â¬ĩ": 39370, + "â¬ĩï¸ı": 25621, + "â¬ĩï¸ı": 13984, + "â¬ĩï¸ıâ¬ĩï¸ı": 40159, + "âĢ": 728, + "âĢ¢": 9485, + "âĢ¢": 2701, + "âĢ¢âĢ¢": 15006, + "âĢ¢âĢ¢": 47575, + "âĢ¢âĢ¢âĢ¢âĢ¢": 27502, + "âĢ¢âĢ¢âĢ¢âĢ¢âĢ¢âĢ¢âĢ¢âĢ¢": 48630, + "âĢ¦": 7095, + "âĢ¦\"": 20215, + "âĢ¦..": 47779, + "âĢ¦.": 18615, + "âĢ¦/": 29842, + "âĢ¦": 959, + "âĢ¦âĢ¦": 40066, + "âĢ²": 32633, + "âĢ³": 25061, + "âĢ¼": 6578, + "âĢ¼ï¸ı": 15622, + "âĢ¼ï¸ı": 8310, + "âĢ¼ï¸ıâĢ¼ï¸ı": 33218, + "âĢĭ": 17086, + "âĢĭ": 9844, + "âĢį": 4244, + "âĢįâĻ": 5177, + "âĢįâĻĢï¸ı": 18897, + "âĢįâĻĢï¸ı": 9605, + "âĢįâĻĤ": 8832, + "âĢįâĻĤï¸ı": 21779, + "âĢįâĻĤï¸ı": 10613, + "âĢİ": 31001, + "âĢIJ": 34512, + "âĢĵ": 21070, + "âĢĵ": 1224, + "âĢĶ": 6718, + "âĢĶ": 2005, + "âĢĶ>": 26341, + "âĢĶ@": 28470, + "âĢĶâĢĶ": 10037, + "âĢĶâĢĶ": 44800, + "âĢĶâĢĶâĢĶâĢĶ": 17797, + "âĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶ": 34432, + "âĢķ": 14236, + "âģ": 1667, + "âģ£": 31089, + "âģ£": 16845, + "âģ¦": 2773, + "âģ¦": 34855, + "âģ¦@": 2859, + "âģ¦âģ¦@": 27783, + "âģ©": 20097, + "âģ©,": 48749, + "âģ©.": 35777, + "âģ©": 2918, + "âģīï¸ı": 46534, + "âģł": 23881, + "âģł": 13503, + "âģłâģł": 33488, + "âĤ": 5227, + "âĤ¬": 34919, + "âĤ¬": 6309, + "âĤ¹": 21777, + "âĥ": 2805, + "âĥ£": 11250, + "âĥ£": 3076, + "âĥ£@": 48291, + "âĦ": 8604, + "âĦ¢": 29438, + "âĦ¢": 11675, + "âĦ¹": 45462, + "âĨ": 6059, + "âĨĴ": 7481, + "âĨĵ": 41603, + "âĩ": 27228, + "âĪ": 17788, + "âī": 22684, + "âīĪ": 45451, + "âĮ": 17848, + "âĮļ": 31301, + "âĮļï¸ı": 35931, + "âı": 7960, + "âı©": 40847, + "âı°": 12714, + "âı±": 33149, + "âı³": 47617, + "âĵ": 27400, + "âĶ": 13389, + "âĶĢ": 45139, + "âĶģ": 42022, + "âķ": 17027, + "âķIJ": 48039, + "âĸ": 4168, + "âĸª": 21203, + "âĸª": 36628, + "âĸªï¸ı": 24974, + "âĸ«": 39478, + "âĸ¬": 33798, + "âĸ¬âĸ¬": 36975, + "âĸ¶": 12509, + "âĸ¶": 21126, + "âĸ¶ï¸ı": 14442, + "âĸº": 46061, + "âĸº": 12086, + "âĸ½": 45634, + "âĸł": 36791, + "âĹ": 9323, + "âĹĨ": 48961, + "âĹı": 26999, + "âĺ": 1741, + "âĺ®": 45851, + "âĺ¹": 28811, + "âĺ¹ï¸ı": 39605, + "âĺº": 5010, + "âĺº": 8703, + "âĺºâĺº": 46051, + "âĺºï¸ı": 11506, + "âĺºï¸ı": 7779, + "âĺºï¸ıâĺºï¸ı": 41315, + "âĺ¼": 38877, + "âĺĢ": 32146, + "âĺĢ": 22242, + "âĺĢï¸ı": 12817, + "âĺĢï¸ı": 8219, + "âĺĢï¸ıâĺĢï¸ı": 44550, + "âĺģ": 25195, + "âĺģï¸ı": 35197, + "âĺĥ": 38972, + "âĺħ": 9339, + "âĺħ": 10643, + "âĺħâĺħ": 12681, + "âĺħâĺħ": 36644, + "âĺħâĺħâĺħâĺħ": 34431, + "âĺħâĺħâĺħâĺħ": 44034, + "âĺħâĺħâĺħâĺħâĺħ": 45984, + "âĺĨ": 23941, + "âĺĨ": 13439, + "âĺİ": 24045, + "âĺİ": 45493, + "âĺİï¸ı": 27219, + "âĺij": 20983, + "âĺij": 42300, + "âĺijï¸ı": 22291, + "âĺĶï¸ı": 31238, + "âĺķ": 11454, + "âĺķ": 26561, + "âĺķï¸ı": 25839, + "âĺķï¸ı": 15499, + "âĺĺ": 23483, + "âĺĺï¸ı": 31454, + "âĺĿ": 21982, + "âĺĿï¸ı": 38891, + "âĺŀ": 31255, + "âĺłï¸ı": 34672, + "âĻ": 1548, + "âĻ¡": 11091, + "âĻ¡": 6251, + "âĻ¡âĻ¡": 22360, + "âĻ¡âĻ¡": 34267, + "âĻ¡âĻ¡âĻ¡": 36611, + "âĻ¤": 47435, + "âĻ¥": 4622, + "âĻ¥": 3405, + "âĻ¥âĻ¥": 12975, + "âĻ¥âĻ¥": 19604, + "âĻ¥âĻ¥âĻ¥": 23255, + "âĻ¥âĻ¥âĻ¥âĻ¥": 49020, + "âĻ¥ï¸ı": 17774, + "âĻ¥ï¸ı": 10561, + "âĻ¥ï¸ıâĻ¥ï¸ı": 40309, + "âĻ¦": 32376, + "âĻ¦": 47547, + "âĻ©": 30339, + "âĻ©âĻ«": 31636, + "âĻª": 27364, + "âĻª": 12382, + "âĻ«": 39217, + "âĻ«": 10814, + "âĻ¬": 24753, + "âĻ»": 39611, + "âĻ»ï¸ı": 46075, + "âļ": 2234, + "âļ¡": 40098, + "âļ¡": 20712, + "âļ¡ï¸ı": 19500, + "âļ¡ï¸ı": 11605, + "âļ¡ï¸ıâļ¡ï¸ı": 45922, + "âļª": 11922, + "âļª": 36373, + "âļªï¸ı": 22251, + "âļªï¸ı": 17885, + "âļ«": 15374, + "âļ«ï¸ı": 26529, + "âļ«ï¸ı": 24649, + "âļ½": 4867, + "âļ½": 13173, + "âļ½âļ½": 43259, + "âļ½ï¸ı": 11342, + "âļ½ï¸ı": 6768, + "âļ½ï¸ıâļ½ï¸ı": 30358, + "âļ½ï¸ıâļ½ï¸ı": 44148, + "âļ¾": 11314, + "âļ¾": 34717, + "âļ¾ï¸ı": 24727, + "âļ¾ï¸ı": 14858, + "âļĵ": 23522, + "âļĵï¸ı": 35299, + "âļĶï¸ı": 29361, + "âļľ": 47491, + "âļł": 39203, + "âļłï¸ı": 40966, + "âļłï¸ı": 15596, + "âĽ": 7956, + "âĽ³ï¸ı": 29204, + "âĽĦ": 30668, + "âĽĦï¸ı": 45465, + "âľ": 1508, + "⾨": 7181, + "⾨": 3531, + "⾨⾨": 35174, + "⾨⾨": 21985, + "⾨⾨⾨": 39424, + "âľĤ": 38602, + "âľħ": 29544, + "âľħ": 5564, + "âľĪ": 10682, + "âľĪ": 30712, + "âľĪï¸ı": 26176, + "âľĪï¸ı": 13413, + "âľĬ": 12392, + "âľĬ": 17819, + "âľĬðŁı½": 48547, + "âľĬðŁı¾": 41185, + "âľĭ": 39383, + "âľĭ": 30239, + "âľĮ": 6419, + "âľĮ": 12656, + "âľĮï¸ı": 21906, + "âľĮï¸ı": 12239, + "âľĮðŁı»": 30538, + "âľĮðŁı¼": 30588, + "âľį": 20872, + "âľįï¸ı": 30888, + "âľı": 32574, + "âľıï¸ı": 40724, + "âľĵ": 36700, + "âľĶ": 47200, + "âľĶ": 13749, + "âľĶï¸ı": 40544, + "âľĶï¸ı": 9191, + "âľĸï¸ı": 44133, + "âľĿ": 42220, + "âĿ": 1045, + "âĿ£": 37007, + "âĿ£": 25623, + "âĿ£ï¸ı": 25240, + "âĿ¤": 1266, + "âĿ¤": 2720, + "âĿ¤âĿ¤": 9033, + "âĿ¤âĿ¤": 14058, + "âĿ¤âĿ¤âĿ¤": 16708, + "âĿ¤âĿ¤âĿ¤âĿ¤": 37918, + "âĿ¤âĿ¤âĿ¤âĿ¤": 43970, + "âĿ¤ï¸ı": 2626, + "âĿ¤ï¸ı#": 30281, + "âĿ¤ï¸ı.": 45326, + "âĿ¤ï¸ı": 1752, + "âĿ¤ï¸ı@": 31187, + "âĿ¤ï¸ıâĿ¤ï¸ı": 6713, + "âĿ¤ï¸ıâĿ¤ï¸ı": 10363, + "âĿ¤ï¸ıâĿ¤ï¸ıâĿ¤ï¸ı": 12282, + "âĿ¤ï¸ıâĿ¤ï¸ıâĿ¤ï¸ıâĿ¤ï¸ı": 39167, + "âĿ¤ï¸ıâĿ¤ï¸ıâĿ¤ï¸ıâĿ¤ï¸ı": 29880, + "âĿ¤ï¸ıðŁĴĻ": 37380, + "âĿ¤ï¸ıðŁĺį": 37272, + "âĿ¤ï¸ıðŁĺĺ": 41800, + "âĿ¤ðŁĺį": 49120, + "âĿ¥": 36914, + "âĿĦ": 8501, + "âĿĦ": 30494, + "âĿĦï¸ı": 16834, + "âĿĦï¸ı": 12402, + "âĿĦï¸ıâĿĦï¸ı": 41626, + "âĿĮ": 44485, + "âĿĮ": 17975, + "âĿĵ": 29791, + "âĿĹ": 12868, + "âĿĹ": 29079, + "âĿĹï¸ı": 28642, + "âĿĹï¸ı": 17391, + "âĿĿ": 46951, + "âŀ": 3257, + "âŀ¡": 12854, + "âŀ¡ï¸ı": 31860, + "âŀ¡ï¸ı": 4956, + "âŀ¤": 18651, + "âŀķ": 46526, + "âŀĸ": 21327, + "âŀĸ": 34902, + "âŀĸâŀĸ": 23316, + "âŀĸâŀĸâŀĸâŀĸ": 40401, + "âŀľ": 23775, + "âł": 5689, + "âłĢ": 9691, + "âłĢ": 8621, + "âłĢâłĢ": 11466, + "âłĢâłĢ": 39092, + "âłĢâłĢâłĢâłĢ": 20976, + "âłĢâłĢâłĢâłĢâłĢâłĢâłĢâłĢ": 46063, + "âŃ": 5527, + "âŃIJ": 6410, + "âŃIJ": 19012, + "âŃIJâŃIJ": 32663, + "âŃIJï¸ı": 12427, + "âŃIJï¸ı": 10251, + "âŃIJï¸ıâŃIJï¸ı": 18640, + "âŃIJï¸ıâŃIJï¸ıâŃIJï¸ı": 40746, + "ã": 159, + "ã": 415, + "ãĢ": 4092, + "ãĢģ": 45262, + "ãĢĤ": 38060, + "ãĢĤ": 38000, + "ãĢĬ": 39920, + "ãĢĭ": 32898, + "ãĢĮ": 18116, + "ãĢį": 19149, + "ãĢİ": 26947, + "ãĢı": 30293, + "ãĢIJ": 12534, + "ãĢij": 12990, + "ãĢľ": 39581, + "ãģ": 4813, + "ãģ¦": 48029, + "ãģ¨": 34671, + "ãģ¨ç¹ĭãģ": 47310, + "ãģ¨ç¹ĭãģĮãĤĬãģŁãģĦ": 48290, + "ãģª": 29104, + "ãģ®": 21575, + "ãģ·": 44130, + "ãģĦ": 33523, + "ãģĦ": 38850, + "ãģĨ": 44235, + "ãģį": 42184, + "ãĤ": 3909, + "ãĤ¢": 26560, + "ãĤ¤": 19319, + "ãĤ¤ãĥ": 36294, + "ãĤ«": 37367, + "ãĤ¯": 31574, + "ãĤ·": 37665, + "ãĤ¸": 32234, + "ãĤ¸ãĥ": 43491, + "ãĤ¹": 22694, + "ãĤ¹": 39220, + "ãĤ¹ãĥ": 32421, + "ãĤ¿": 34941, + "ãĤĬãģ": 40500, + "ãĤĮ": 45211, + "ãĤŃ": 47121, + "ãĥ": 2429, + "ãĥ©": 23007, + "ãĥª": 32115, + "ãĥ«": 33257, + "ãĥ¬": 32965, + "ãĥ³": 17671, + "ãĥ³": 26875, + "ãĥ³ãĤ": 45105, + "ãĥ³ãĥ": 25914, + "ãĥ»": 8415, + "ãĥ»": 11158, + "ãĥ»ãĥ»": 13949, + "ãĥ»ãĥ»ãĥ»": 14234, + "ãĥ¼": 13457, + "ãĥ¼": 30391, + "ãĥ¼ãĥ": 18584, + "ãĥĥ": 28902, + "ãĥĦ": 32173, + "ãĥĪ": 42384, + "ãĥİ": 39967, + "ãĥķãĤ": 33371, + "ãĥŀ": 48924, + "ãĥŃ": 35827, + "ãħ": 5947, + "ãħ¤": 21096, + "ãħ¤ãħ¤": 22583, + "ãħ¤ãħ¤ãħ¤ãħ¤": 39329, + "ãħĭ": 13052, + "ãħĭ": 25108, + "ãħĭãħĭ": 16604, + "ãħĭãħĭ": 42581, + "ãħĭãħĭãħĭ": 46407, + "ãħĭãħĭãħĭãħĭ": 39362, + "ãħł": 16089, + "ãħł": 25781, + "ãħłãħł": 22021, + "ãħłãħł": 34398, + "ãħłãħłãħłãħł": 47028, + "ä": 160, + "ä": 416, + "ä¸": 19759, + "ä¹": 41854, + "äº": 21078, + "人": 36839, + "ä»": 37743, + "ä½": 47466, + "å": 161, + "å": 417, + "å¤": 23170, + "å¥": 29290, + "å®": 27047, + "å°": 34720, + "å±": 46096, + "å¸": 42021, + "å¹": 38780, + "åħ": 34314, + "åĨ": 27972, + "åĨĻ": 44653, + "åĪ": 42748, + "åĭ": 47505, + "åı": 34517, + "åIJ": 41673, + "åĽ": 39027, + "åľ": 37746, + "åŃ": 35751, + "æ": 162, + "æ": 418, + "æĸ": 29032, + "æĹ": 22265, + "æĹ¥": 39121, + "æĹ¥": 37156, + "æĺ": 42891, + "æĻ": 48132, + "æľ": 19277, + "æľ¬": 44353, + "æĿ": 27667, + "æĿ±": 48338, + "ç": 163, + "ç": 419, + "ç¥": 26369, + "ç¥Ń": 42557, + "çµ": 37810, + "ç¹": 43431, + "ç¹ĭãģ": 45930, + "çĶ": 20211, + "çĶŁ": 33375, + "çľ": 33440, + "羣": 41570, + "è": 164, + "è": 420, + "èª": 34002, + "èªķ": 41293, + "é": 165, + "é": 421, + "éģ": 44854, + "éĩ": 38283, + "ê": 166, + "ê": 422, + "ê°": 21122, + "ê°ĵ": 41076, + "ê°ĵìĦ¸ë¸IJ": 41689, + "ê°ķ": 45758, + "ê²": 35555, + "ê³": 36216, + "êµ": 31871, + "ê·": 42680, + "ê¸": 32495, + "ê¹": 24531, + "ê¹Ģ": 25203, + "ë": 167, + "ë": 423, + "ë¦": 24621, + "리": 47649, + "ë§": 28024, + "ë§Ī": 40027, + "ëª": 36311, + "ë¯": 19528, + "민": 34442, + "민": 44632, + "ë°": 15810, + "ë°©": 23273, + "ë°©íĥ": 25081, + "ë°©íĥĦ": 25641, + "ë°©íĥĦìĨĮëħĦëĭ": 26068, + "ë°©íĥĦìĨĮëħĦëĭ¨": 27129, + "ë°ķ": 40988, + "ë²": 48267, + "ë³": 44693, + "ë¹": 24193, + "ëĤ": 27252, + "ëĤĺ": 48484, + "ëĭ": 13094, + "ëĭ¤": 46680, + "ëĭĪ": 33708, + "ëį": 45543, + "ëı": 31972, + "ëĵ": 30850, + "ëĿ": 44317, + "ì": 168, + "ì": 424, + "ì£": 39856, + "주": 45161, + "ì¤": 31153, + "ì§": 16279, + "ì§Ģ": 28836, + "ì§Ħ": 38890, + "ì°": 40742, + "ì¶": 42476, + "ì¶ķ": 46403, + "ì¶ķíķĺ": 47866, + "ì¹": 45088, + "ìĤ": 31061, + "ìĥ": 30587, + "ìĥĿ": 47858, + "ìĦ": 15074, + "ìĦ¸ë": 29254, + "ìĦ¸ë¸": 29658, + "ìĦ¸ë¸IJ": 41415, + "ìĨ": 15115, + "ìĨĮë": 20515, + "ìĨĮëħ": 21391, + "ìĨĮëħĦëĭ": 25887, + "ìĪ": 32757, + "ìĬ": 12125, + "ìĬ¤": 20305, + "ìĬ¤": 23829, + "ìĭ": 23924, + "ìķ": 16071, + "ìķĦ": 23233, + "ìĸ": 31625, + "ìĹ": 13252, + "ìĹIJ": 37622, + "ìĹij": 31036, + "ìĹijìĨ": 42763, + "ìĹijìĨĮ": 45606, + "ìĺ": 21144, + "ìĻ": 39405, + "ìļ": 18541, + "ìļ°": 38415, + "ìļ°": 49344, + "ìĽ": 22543, + "ìĽIJ": 36495, + "ìľ": 20909, + "ìľł": 42890, + "ìĿ": 8276, + "ìĿ´": 12286, + "ìĿ´": 34746, + "ìĿ´ì": 37590, + "ìĿ¼": 43406, + "ìŀ": 20849, + "ìł": 20580, + "ìłķ": 34725, + "í": 169, + "í": 425, + "íģ": 35641, + "íģ¬": 45832, + "íĤ": 43565, + "íĥ": 15012, + "íĥĢ": 41126, + "íĥľ": 37663, + "íĬ": 23215, + "íĬ¸": 48974, + "íĬ¸": 39820, + "íĭ": 34350, + "íĶ": 29450, + "íķ": 15197, + "íķ´": 35286, + "íķĺ": 33992, + "íĺ": 15962, + "íĺ¸": 39657, + "íĺĦ": 34645, + "íĻ": 31882, + "î": 170, + "î": 426, + "îĢ": 36288, + "îĦ": 35368, + "îĮ": 41006, + "îIJ": 16929, + "îIJĴ": 40100, + "ï": 171, + "ï": 427, + "ï¸": 842, + "ï¸İ": 24029, + "ï¸ı": 1392, + "ï¸ı#": 46997, + "ï¸ı:": 32604, + "ï¸ı": 1001, + "ï¸ı@": 34600, + "ï¸ıâĥ£": 17394, + "ï¸ıâĥ£-": 40376, + "ï¸ıâĥ£": 4603, + "ï¿": 27850, + "�": 47356, + "�": 39802, + "ð": 172, + "ð": 428, + "ðĿ": 6874, + "ðĿIJ": 15889, + "ðĿij": 43794, + "ðĿĴ": 43387, + "ðĿĵ": 47110, + "ðĿĹ": 18865, + "ðĿĺ": 26109, + "ðĿĻ": 29415, + "ðŁ": 558, + "ðŁ¤": 1793, + "ðŁ¤£": 9665, + "ðŁ¤£": 9909, + "ðŁ¤£ðŁ¤£": 16430, + "ðŁ¤£ðŁ¤£": 31009, + "ðŁ¤£ðŁ¤£ðŁ¤£": 32262, + "ðŁ¤¤": 39550, + "ðŁ¤¤": 26759, + "ðŁ¤¦": 17186, + "ðŁ¤§": 40983, + "ðŁ¤©": 27351, + "ðŁ¤©": 16074, + "ðŁ¤ª": 44230, + "ðŁ¤ª": 24920, + "ðŁ¤«": 47671, + "ðŁ¤¯": 37595, + "ðŁ¤·": 13185, + "ðŁ¤·ðŁı»âĢįâĻĢï¸ı": 46770, + "ðŁ¤ij": 34801, + "ðŁ¤ĵ": 36580, + "ðŁ¤ĵ": 18928, + "ðŁ¤Ķ": 12706, + "ðŁ¤Ķ": 6497, + "ðŁ¤ĶðŁ¤Ķ": 28490, + "ðŁ¤ĶðŁ¤ĶðŁ¤Ķ": 43361, + "ðŁ¤ĸ": 46146, + "ðŁ¤Ĺ": 16646, + "ðŁ¤Ĺ": 10465, + "ðŁ¤ĹðŁ¤Ĺ": 44321, + "ðŁ¤ĺ": 10623, + "ðŁ¤ĺ": 17288, + "ðŁ¤ĺðŁı»": 46449, + "ðŁ¤ĺðŁı»": 30891, + "ðŁ¤ĺðŁı¼": 31458, + "ðŁ¤ĺðŁı½": 49362, + "ðŁ¤Ļ": 23800, + "ðŁ¤Ļ": 39101, + "ðŁ¤Ŀ": 35242, + "ðŁ¤ŀ": 29463, + "ðŁ¤ŀ": 38597, + "ðŁ¤Ł": 48509, + "ðŁ¤ł": 36737, + "ðŁ¤Ń": 47289, + "ðŁ¥": 4156, + "ðŁ¥°": 29246, + "ðŁ¥°": 17597, + "ðŁ¥³": 45823, + "ðŁ¥³": 28055, + "ðŁ¥º": 43380, + "ðŁ¥º": 36858, + "ðŁ¥Ĥ": 43805, + "ðŁ¥Ĥ": 25212, + "ðŁ¥ĥ": 47790, + "ðŁ¥ĩ": 34372, + "ðŁ¥ĩ": 20069, + "ðŁ¥Ī": 35858, + "ðŁ¥ī": 36782, + "ðŁ¥Ĭ": 29275, + "ðŁ¦": 6040, + "ðŁ¦ģ": 36367, + "ðŁ¦ģ": 26056, + "ðŁ¦ĥ": 40184, + "ðŁ¦Ħ": 37659, + "ðŁ¦ħ": 28800, + "ðŁ¦Ī": 48984, + "ðŁ¦ĭ": 49325, + "ðŁ¦ĭ": 28985, + "ðŁ§": 8792, + "ðŁ§¡": 30996, + "ðŁ§¡": 24578, + "ðŁ§IJ": 33549, + "ðŁħ": 22010, + "ðŁĨ": 9536, + "ðŁĨķ": 34956, + "ðŁĨĺ": 39868, + "ðŁĨļ": 16325, + "ðŁĩ": 1173, + "ðŁĩ¦": 12469, + "ðŁĩ¦": 28565, + "ðŁĩ¦ðŁĩ": 33196, + "ðŁĩ¦ðŁĩ·": 41629, + "ðŁĩ¦ðŁĩº": 25192, + "ðŁĩ§": 14660, + "ðŁĩ§ðŁĩ": 37342, + "ðŁĩ§ðŁĩª": 38794, + "ðŁĩ§ðŁĩ·": 28182, + "ðŁĩ¨": 8889, + "ðŁĩ¨ðŁĩ": 8989, + "ðŁĩ¨ðŁĩ¦": 34324, + "ðŁĩ¨ðŁĩ¦": 16364, + "ðŁĩ¨ðŁĩ³": 36819, + "ðŁĩ¨ðŁĩŃ": 41119, + "ðŁĩ©": 15222, + "ðŁĩ©ðŁĩ": 36350, + "ðŁĩ©ðŁĩª": 21531, + "ðŁĩª": 11428, + "ðŁĩª": 12331, + "ðŁĩªðŁĩ": 13917, + "ðŁĩªðŁĩ¸": 22177, + "ðŁĩªðŁĩº": 34655, + "ðŁĩ«": 12977, + "ðŁĩ«ðŁĩ·": 39109, + "ðŁĩ«ðŁĩ·": 16223, + "ðŁĩ¬": 8129, + "ðŁĩ¬ðŁĩ": 8354, + "ðŁĩ¬ðŁĩ§": 23762, + "ðŁĩ¬ðŁĩ§": 11559, + "ðŁĩ®": 8268, + "ðŁĩ®ðŁĩ": 8347, + "ðŁĩ®ðŁĩª": 34148, + "ðŁĩ®ðŁĩ³": 47299, + "ðŁĩ®ðŁĩ³": 23602, + "ðŁĩ®ðŁĩ¹": 42034, + "ðŁĩ®ðŁĩ¹": 17070, + "ðŁĩ¯": 20090, + "ðŁĩ¯ðŁĩ": 22924, + "ðŁĩ¯ðŁĩµ": 26527, + "ðŁĩ°": 28232, + "ðŁĩ±": 29533, + "ðŁĩ±ðŁĩ": 40941, + "ðŁĩ²": 16411, + "ðŁĩ²ðŁĩ": 17562, + "ðŁĩ²ðŁĩ½": 32073, + "ðŁĩ³": 16645, + "ðŁĩ³ðŁĩ": 17747, + "ðŁĩ³ðŁĩ±": 36747, + "ðŁĩµ": 12127, + "ðŁĩµðŁĩ": 13608, + "ðŁĩµðŁĩ°": 37764, + "ðŁĩµðŁĩ¹": 42621, + "ðŁĩµðŁĩŃ": 42777, + "ðŁĩ·": 16026, + "ðŁĩ·": 9869, + "ðŁĩ·ðŁĩº": 37902, + "ðŁĩ¸": 19447, + "ðŁĩ¸ðŁĩ": 33325, + "ðŁĩ¸ðŁĩª": 39260, + "ðŁĩ¹": 21810, + "ðŁĩ¹ðŁĩ": 36250, + "ðŁĩº": 4054, + "ðŁĩº": 17467, + "ðŁĩºðŁĩ": 4131, + "ðŁĩºðŁĩ¸": 8907, + "ðŁĩºðŁĩ¸": 5688, + "ðŁĩºðŁĩ¸ðŁĩºðŁĩ¸": 18739, + "ðŁĩºðŁĩ¸ðŁĩºðŁĩ¸": 41411, + "ðŁĩºðŁĩ¸ðŁĩºðŁĩ¸ðŁĩºðŁĩ¸": 43357, + "ðŁĩ¿": 25520, + "ðŁĩ¿ðŁĩ¦": 36982, + "ðŁĩŃ": 30370, + "ðŁĮ": 1576, + "ðŁĮ±": 35318, + "ðŁĮ±": 20665, + "ðŁĮ²": 34071, + "ðŁĮ²": 28154, + "ðŁĮ³": 44265, + "ðŁĮ³": 28543, + "ðŁĮ´": 20643, + "ðŁĮ´": 15968, + "ðŁĮµ": 40871, + "ðŁĮ·": 32328, + "ðŁĮ·": 24259, + "ðŁĮ¸": 16314, + "ðŁĮ¸": 10980, + "ðŁĮ¸ðŁĮ¸": 46210, + "ðŁĮ¹": 14990, + "ðŁĮ¹": 10662, + "ðŁĮ¹ðŁĮ¹": 37933, + "ðŁĮº": 27608, + "ðŁĮº": 19829, + "ðŁĮ»": 27196, + "ðŁĮ»": 19772, + "ðŁĮ¼": 36484, + "ðŁĮ¼": 26312, + "ðŁĮ¾": 39796, + "ðŁĮ¿": 27736, + "ðŁĮ¿": 18588, + "ðŁĮĢ": 34348, + "ðŁĮħ": 27547, + "ðŁĮĪ": 23038, + "ðŁĮĪ": 13042, + "ðŁĮĬ": 20465, + "ðŁĮĬ": 14302, + "ðŁĮĮ": 43393, + "ðŁĮį": 34931, + "ðŁĮį": 18641, + "ðŁĮİ": 31125, + "ðŁĮİ": 16969, + "ðŁĮı": 31527, + "ðŁĮIJ": 33071, + "ðŁĮĻ": 42330, + "ðŁĮĻ": 23283, + "ðŁĮļ": 49004, + "ðŁĮļ": 27877, + "ðŁĮŀ": 21152, + "ðŁĮŀ": 12980, + "ðŁĮŁ": 13196, + "ðŁĮŁ": 8542, + "ðŁĮŁðŁĮŁ": 26014, + "ðŁį": 2011, + "ðŁį¦": 47375, + "ðŁį¦": 32032, + "ðŁį©": 38379, + "ðŁįª": 38958, + "ðŁį«": 47994, + "ðŁį«": 33401, + "ðŁį°": 43732, + "ðŁį°": 30051, + "ðŁį³": 37441, + "ðŁį´": 41531, + "ðŁį´": 25338, + "ðŁį·": 24445, + "ðŁį·": 18072, + "ðŁį¸": 43058, + "ðŁį¸": 31217, + "ðŁį¹": 35598, + "ðŁįº": 31081, + "ðŁįº": 21590, + "ðŁį»": 22793, + "ðŁį»": 13167, + "ðŁį¾": 27294, + "ðŁį¾": 21656, + "ðŁįĢ": 22865, + "ðŁįĢ": 15764, + "ðŁįģ": 29837, + "ðŁįģ": 23075, + "ðŁįĤ": 35015, + "ðŁįĤ": 25721, + "ðŁįĥ": 27157, + "ðŁįĥ": 20147, + "ðŁįĩ": 48697, + "ðŁįĬ": 35001, + "ðŁįĬ": 28036, + "ðŁįĭ": 39543, + "ðŁįĮ": 44987, + "ðŁįį": 48946, + "ðŁįİ": 32069, + "ðŁįij": 32889, + "ðŁįĴ": 33160, + "ðŁįĵ": 44739, + "ðŁįĵ": 33456, + "ðŁįĶ": 46415, + "ðŁįĶ": 36031, + "ðŁįķ": 31469, + "ðŁįķ": 23904, + "ðŁįŃ": 42100, + "ðŁİ": 1165, + "ðŁİ£": 43158, + "ðŁİ¤": 23490, + "ðŁİ¤": 15690, + "ðŁİ¥": 22186, + "ðŁİ¥:": 43640, + "ðŁİ¥": 13233, + "ðŁİ§": 31254, + "ðŁİ§": 14266, + "ðŁİ¨": 31953, + "ðŁİ¨": 13461, + "ðŁİ©": 37701, + "ðŁİ«": 30331, + "ðŁİ¬": 36020, + "ðŁİ¬": 18150, + "ðŁİ®": 29312, + "ðŁİ¯": 23114, + "ðŁİµ": 27435, + "ðŁİµ": 14946, + "ðŁİ¶": 11755, + "ðŁİ¶": 6011, + "ðŁİ¶ðŁİ¶": 36283, + "ðŁİ¸": 29135, + "ðŁİ¸": 22122, + "ðŁİ¹": 43493, + "ðŁİ¼": 34949, + "ðŁİ¼": 23757, + "ðŁİ¾": 41982, + "ðŁİ¾": 24222, + "ðŁİĢ": 34347, + "ðŁİĢ": 20151, + "ðŁİģ": 18368, + "ðŁİģ": 13462, + "ðŁİĤ": 13026, + "ðŁİĤ": 10392, + "ðŁİĤðŁİĤ": 39338, + "ðŁİĥ": 22622, + "ðŁİĥ": 16780, + "ðŁİĦ": 12942, + "ðŁİĦ": 11267, + "ðŁİħ": 17685, + "ðŁİħ": 24276, + "ðŁİĨ": 39222, + "ðŁİĪ": 16142, + "ðŁİĪ": 14448, + "ðŁİĪðŁİī": 48049, + "ðŁİī": 4310, + "ðŁİī:": 17310, + "ðŁİī": 3986, + "ðŁİīðŁİ": 11473, + "ðŁİīðŁİĪ": 40499, + "ðŁİīðŁİĪ": 34008, + "ðŁİīðŁİī": 25159, + "ðŁİīðŁİī": 13450, + "ðŁİīðŁİīðŁİī": 20828, + "ðŁİīðŁİĬ": 31662, + "ðŁİīðŁİĬ": 30781, + "ðŁİĬ": 22763, + "ðŁİĬ": 22425, + "ðŁİĬðŁİī": 48801, + "ðŁİĵ": 28916, + "ðŁİĵ": 18744, + "ðŁİĻ": 29001, + "ðŁİĻ": 29753, + "ðŁİĻï¸ı": 44205, + "ðŁİŁ": 19248, + "ðŁİŁ": 21107, + "ðŁİŁï¸ı": 30243, + "ðŁİŃ": 28856, + "ðŁı": 1109, + "ðŁı¡": 27318, + "ðŁı³ï¸ı": 26844, + "ðŁı³ï¸ıâĢį": 27093, + "ðŁı³ï¸ıâĢįðŁĮĪ": 32610, + "ðŁı´": 39690, + "ðŁı´": 19704, + "ðŁı»": 5042, + "ðŁı»": 3702, + "ðŁı»âĢį": 46250, + "ðŁı»âĢįâĻĢï¸ı": 48391, + "ðŁı»âĢįâĻĢï¸ı": 23595, + "ðŁı»âĢįâĻĤï¸ı": 30984, + "ðŁı¼": 6193, + "ðŁı¼": 4027, + "ðŁı¼âĢįâĻĢï¸ı": 28955, + "ðŁı½": 8514, + "ðŁı½": 6114, + "ðŁı½âĢįâĻĢï¸ı": 37036, + "ðŁı½âĢįâĻĤï¸ı": 43157, + "ðŁı¾": 10230, + "ðŁı¾": 7778, + "ðŁı¾âĢįâĻĤï¸ı": 47189, + "ðŁı¿": 29854, + "ðŁı¿": 21094, + "ðŁıĢ": 13708, + "ðŁıĢ": 8813, + "ðŁıĢðŁıĢ": 43169, + "ðŁıģ": 29423, + "ðŁıģ": 17473, + "ðŁıĥ": 16820, + "ðŁıĥ": 32751, + "ðŁıħ": 25500, + "ðŁıĨ": 9585, + "ðŁıĨ": 5596, + "ðŁıĨðŁıĨ": 18946, + "ðŁıĨðŁıĨ": 38269, + "ðŁıĨðŁıĨðŁıĨ": 44484, + "ðŁıĩ": 45789, + "ðŁıĩ": 40288, + "ðŁıĪ": 16144, + "ðŁıĪ": 10477, + "ðŁıī": 26020, + "ðŁıĬ": 33061, + "ðŁıĬ": 47830, + "ðŁıĮ": 41116, + "ðŁıı": 32460, + "ðŁıIJ": 46334, + "ðŁıIJ": 29433, + "ðŁıĴ": 37756, + "ðŁıŁ": 35914, + "ðŁıŁ": 26472, + "ðŁıŁï¸ı": 42627, + "ðŁıł": 33727, + "ðŁIJ": 2074, + "ðŁIJ¢": 37049, + "ðŁIJ£": 39597, + "ðŁIJ¥": 42981, + "ðŁIJ¦": 37260, + "ðŁIJ¬": 44238, + "ðŁIJ¯": 34825, + "ðŁIJ¯": 26111, + "ðŁIJ°": 35378, + "ðŁIJ°": 25050, + "ðŁIJ±": 35710, + "ðŁIJ±": 22979, + "ðŁIJ´": 33509, + "ðŁIJ¶": 14466, + "ðŁIJ¶": 10631, + "ðŁIJ·": 38408, + "ðŁIJ¸": 45597, + "ðŁIJ¸": 40298, + "ðŁIJº": 44281, + "ðŁIJº": 31445, + "ðŁIJ»": 30750, + "ðŁIJ»": 25322, + "ðŁIJ¼": 46234, + "ðŁIJ¾": 16057, + "ðŁIJ¾": 11317, + "ðŁIJ¾ðŁIJ¾": 42202, + "ðŁIJī": 46908, + "ðŁIJĬ": 43974, + "ðŁIJį": 48903, + "ðŁIJį": 30177, + "ðŁIJİ": 48281, + "ðŁIJİ": 32726, + "ðŁIJIJ": 47735, + "ðŁIJIJ": 27954, + "ðŁIJij": 49389, + "ðŁIJķ": 41069, + "ðŁIJĺ": 38733, + "ðŁIJĿ": 30619, + "ðŁIJĿ": 20111, + "ðŁIJŁ": 42084, + "ðŁIJŁ": 29989, + "ðŁIJł": 42725, + "ðŁij": 964, + "ðŁij£": 39755, + "ðŁij§": 48938, + "ðŁij¨": 18966, + "ðŁij¨âĢį": 25023, + "ðŁij©": 18800, + "ðŁij©âĢį": 26304, + "ðŁij«": 47106, + "ðŁij«": 35457, + "ðŁij®": 42686, + "ðŁij¯": 25910, + "ðŁij¯": 20582, + "ðŁij¶": 26187, + "ðŁij¶": 33189, + "ðŁij¸": 26268, + "ðŁij¸": 36645, + "ðŁij¹": 46766, + "ðŁij»": 24625, + "ðŁij»": 16243, + "ðŁij¼": 25270, + "ðŁij¼": 31083, + "ðŁij½": 42677, + "ðŁij½": 26257, + "ðŁijĢ": 11524, + "ðŁijĢ": 5908, + "ðŁijĢðŁijĢ": 31561, + "ðŁijģ": 47796, + "ðŁijģ": 45705, + "ðŁijĦ": 47445, + "ðŁijħ": 31833, + "ðŁijħ": 24672, + "ðŁijĨ": 42975, + "ðŁijĨ": 45194, + "ðŁijĩ": 7662, + "ðŁijĩ": 7475, + "ðŁijĩðŁı»": 45811, + "ðŁijĩðŁı»": 32813, + "ðŁijĩðŁı¼": 37504, + "ðŁijĩðŁijĩ": 17915, + "ðŁijĩðŁijĩ": 31891, + "ðŁijĩðŁijĩðŁijĩ": 35627, + "ðŁijĪ": 32794, + "ðŁijĪ": 20832, + "ðŁijī": 9477, + "ðŁijī": 3988, + "ðŁijīðŁı»": 23481, + "ðŁijīðŁı¼": 27534, + "ðŁijīðŁı½": 38059, + "ðŁijīðŁijī": 41480, + "ðŁijĬ": 8897, + "ðŁijĬ": 9704, + "ðŁijĬðŁı»": 47393, + "ðŁijĬðŁı»": 29152, + "ðŁijĬðŁı¼": 49000, + "ðŁijĬðŁı¼": 30115, + "ðŁijĬðŁijĬ": 46521, + "ðŁijĭ": 19351, + "ðŁijĭ": 17686, + "ðŁijĮ": 4890, + "ðŁijĮ": 4494, + "ðŁijĮðŁı»": 31818, + "ðŁijĮðŁı»": 18606, + "ðŁijĮðŁı¼": 37655, + "ðŁijĮðŁı¼": 20031, + "ðŁijĮðŁı½": 35834, + "ðŁijĮðŁijĮ": 36139, + "ðŁijĮðŁijĮ": 21435, + "ðŁijĮðŁijĮðŁijĮ": 40876, + "ðŁijį": 4686, + "ðŁijį": 4201, + "ðŁijįðŁı»": 25803, + "ðŁijįðŁı»": 15129, + "ðŁijįðŁı¼": 37285, + "ðŁijįðŁı¼": 19689, + "ðŁijįðŁı½": 43722, + "ðŁijįðŁijį": 33012, + "ðŁijįðŁijį": 18997, + "ðŁijįðŁijįðŁijį": 37284, + "ðŁijİ": 39702, + "ðŁijİ": 32568, + "ðŁijı": 3802, + "ðŁijı": 4829, + "ðŁijıðŁı»": 19236, + "ðŁijıðŁı»": 17029, + "ðŁijıðŁı»ðŁijıðŁı»": 35254, + "ðŁijıðŁı¼": 24496, + "ðŁijıðŁı¼": 19979, + "ðŁijıðŁı¼ðŁijıðŁı¼": 46712, + "ðŁijıðŁı½": 40796, + "ðŁijıðŁı½": 33978, + "ðŁijıðŁı¾": 45450, + "ðŁijıðŁijı": 10356, + "ðŁijıðŁijı": 16706, + "ðŁijıðŁijıðŁijı": 17254, + "ðŁijIJ": 40877, + "ðŁijij": 14955, + "ðŁijij": 8717, + "ðŁijijðŁijij": 48532, + "ðŁijķ": 47865, + "ðŁijŁ": 41183, + "ðŁijł": 41264, + "ðŁijŃ": 34175, + "ðŁijŃ": 27943, + "ðŁĴ": 837, + "ðŁĴ¡": 24081, + "ðŁĴ£": 36862, + "ðŁĴ£": 29006, + "ðŁĴ¤": 34706, + "ðŁĴ¤": 25632, + "ðŁĴ¥": 12209, + "ðŁĴ¥": 7347, + "ðŁĴ¥ðŁĴ¥": 27396, + "ðŁĴ¥ðŁĴ¥": 39246, + "ðŁĴ¥ðŁĴ¥ðŁĴ¥": 48890, + "ðŁĴ¦": 21180, + "ðŁĴ¦": 14060, + "ðŁĴ¦ðŁĴ¦": 44469, + "ðŁĴ§": 34095, + "ðŁĴ¨": 27408, + "ðŁĴ¨": 17891, + "ðŁĴ©": 48621, + "ðŁĴ©": 28847, + "ðŁĴª": 5475, + "ðŁĴª": 6440, + "ðŁĴªðŁı»": 31669, + "ðŁĴªðŁı»": 21903, + "ðŁĴªðŁı¼": 32041, + "ðŁĴªðŁı¼": 20759, + "ðŁĴªðŁı½": 46380, + "ðŁĴªðŁı½": 31111, + "ðŁĴªðŁı¾": 39398, + "ðŁĴªðŁĴª": 24747, + "ðŁĴªðŁĴªðŁĴª": 39913, + "ðŁĴ«": 25770, + "ðŁĴ«": 12526, + "ðŁĴ¬": 30947, + "ðŁĴ¯": 10611, + "ðŁĴ¯": 7018, + "ðŁĴ¯ðŁĴ¯": 30234, + "ðŁĴ¯ðŁĴ¯": 44070, + "ðŁĴ°": 20454, + "ðŁĴ°": 14078, + "ðŁĴ°ðŁĴ°": 41747, + "ðŁĴµ": 47412, + "ðŁĴµ": 38041, + "ðŁĴ¸": 37696, + "ðŁĴ¸": 25957, + "ðŁĴ»": 33433, + "ðŁĴ»": 18135, + "ðŁĴ¿": 39541, + "ðŁĴĢ": 14888, + "ðŁĴĢ": 12158, + "ðŁĴĢðŁĴĢ": 30884, + "ðŁĴģ": 13997, + "ðŁĴģ": 14392, + "ðŁĴĥ": 9947, + "ðŁĴĥ": 14333, + "ðŁĴĥðŁı»": 38624, + "ðŁĴĥðŁĴĥ": 28041, + "ðŁĴĦ": 46116, + "ðŁĴĦ": 34571, + "ðŁĴħ": 27457, + "ðŁĴħ": 32414, + "ðŁĴī": 44316, + "ðŁĴī": 30503, + "ðŁĴĭ": 12217, + "ðŁĴĭ": 7417, + "ðŁĴĭðŁĴĭ": 29214, + "ðŁĴĮ": 40817, + "ðŁĴį": 35850, + "ðŁĴį": 24898, + "ðŁĴİ": 25938, + "ðŁĴİ": 15874, + "ðŁĴIJ": 27375, + "ðŁĴIJ": 20554, + "ðŁĴij": 49404, + "ðŁĴĵ": 20628, + "ðŁĴĵ": 12568, + "ðŁĴĵðŁĴĵ": 43505, + "ðŁĴĶ": 18880, + "ðŁĴĶ": 10704, + "ðŁĴĶðŁĴĶ": 44673, + "ðŁĴķ": 5412, + "ðŁĴķ": 3082, + "ðŁĴķðŁĴķ": 23106, + "ðŁĴķðŁĴķ": 14117, + "ðŁĴķðŁĴķðŁĴķ": 26772, + "ðŁĴĸ": 8466, + "ðŁĴĸ": 5582, + "ðŁĴĸðŁĴĸ": 19562, + "ðŁĴĸðŁĴĸ": 30595, + "ðŁĴĸðŁĴĸðŁĴĸ": 33915, + "ðŁĴĹ": 10148, + "ðŁĴĹ": 6690, + "ðŁĴĹðŁĴĹ": 47158, + "ðŁĴĹðŁĴĹ": 24064, + "ðŁĴĹðŁĴĹðŁĴĹ": 36990, + "ðŁĴĺ": 18223, + "ðŁĴĺ": 10816, + "ðŁĴĺðŁĴĺ": 40464, + "ðŁĴĻ": 5305, + "ðŁĴĻ": 4074, + "ðŁĴĻðŁĴĻ": 17833, + "ðŁĴĻðŁĴĻ": 27101, + "ðŁĴĻðŁĴĻðŁĴĻ": 30698, + "ðŁĴĻðŁĴĽ": 46804, + "ðŁĴĻðŁĴĽ": 26230, + "ðŁĴĻðŁĴľ": 47931, + "ðŁĴĻðŁĴľ": 42541, + "ðŁĴļ": 8102, + "ðŁĴļ": 6521, + "ðŁĴļðŁĴļ": 27497, + "ðŁĴļðŁĴļ": 46209, + "ðŁĴļðŁĴļðŁĴļ": 46182, + "ðŁĴļðŁĴĽ": 41232, + "ðŁĴĽ": 8221, + "ðŁĴĽ": 6233, + "ðŁĴĽðŁĴĻ": 36337, + "ðŁĴĽðŁĴļ": 37994, + "ðŁĴĽðŁĴĽ": 32420, + "ðŁĴľ": 6832, + "ðŁĴľ": 4882, + "ðŁĴľðŁĴľ": 17280, + "ðŁĴľðŁĴľ": 28211, + "ðŁĴľðŁĴľðŁĴľ": 31004, + "ðŁĴĿ": 36761, + "ðŁĴĿ": 22002, + "ðŁĴŀ": 14862, + "ðŁĴŀ": 8988, + "ðŁĴŀðŁĴŀ": 36448, + "ðŁĴŁ": 49394, + "ðŁĴŁ": 28828, + "ðŁĴŃ": 33848, + "ðŁĵ": 1497, + "ðŁĵ¢": 46560, + "ðŁĵ¢": 20901, + "ðŁĵ£": 48841, + "ðŁĵ£": 21282, + "ðŁĵ°:": 28952, + "ðŁĵ°": 14985, + "ðŁĵ±": 36104, + "ðŁĵ±": 20824, + "ðŁĵ²": 19363, + "ðŁĵ·": 6966, + "ðŁĵ·:": 8294, + "ðŁĵ·": 5551, + "ðŁĵ·@": 40032, + "ðŁĵ¸": 8401, + "ðŁĵ¸:": 10379, + "ðŁĵ¸": 6074, + "ðŁĵ¸@": 39660, + "ðŁĵ¹": 49251, + "ðŁĵº": 21792, + "ðŁĵº:": 29728, + "ðŁĵº": 10450, + "ðŁĵ»": 32711, + "ðŁĵ»": 15882, + "ðŁĵ½": 45361, + "ðŁĵħ": 21277, + "ðŁĵĨ": 23471, + "ðŁĵĪ": 23359, + "ðŁĵĬ": 22244, + "ðŁĵĭ": 46351, + "ðŁĵĮ": 22289, + "ðŁĵį": 25043, + "ðŁĵį:": 36845, + "ðŁĵį": 8903, + "ðŁĵĸ": 49003, + "ðŁĵĸ": 23043, + "ðŁĵļ": 25433, + "ðŁĵļ": 15566, + "ðŁĵĿ": 31888, + "ðŁĵĿ:": 48398, + "ðŁĵĿ": 15853, + "ðŁĵŀ": 24022, + "ðŁĶ": 1428, + "ðŁĶ¥": 3191, + "ðŁĶ¥#": 44354, + "ðŁĶ¥": 3016, + "ðŁĶ¥ðŁĶ¥": 5692, + "ðŁĶ¥ðŁĶ¥": 11771, + "ðŁĶ¥ðŁĶ¥ðŁĶ¥": 11004, + "ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥": 23408, + "ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥": 30989, + "ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥": 48401, + "ðŁĶ¥ðŁĶĹ": 35130, + "ðŁĶª": 47078, + "ðŁĶª": 34545, + "ðŁĶ«": 38116, + "ðŁĶ«": 20583, + "ðŁĶ¬": 44227, + "ðŁĶ®": 38077, + "ðŁĶ´": 12408, + "ðŁĶ´": 10854, + "ðŁĶ´âļªï¸ı": 46879, + "ðŁĶ´âļªï¸ı": 40055, + "ðŁĶµ": 17531, + "ðŁĶµ": 17193, + "ðŁĶµâļªï¸ı": 42412, + "ðŁĶ¶": 42880, + "ðŁĶ¶": 36222, + "ðŁĶ·": 37740, + "ðŁĶ¸": 24200, + "ðŁĶ¹": 19995, + "ðŁĶº": 45561, + "ðŁĶģ": 41299, + "ðŁĶĬ": 32580, + "ðŁĶĬ": 20502, + "ðŁĶİ": 44935, + "ðŁĶij": 35127, + "ðŁĶĴ": 44972, + "ðŁĶĶ": 45753, + "ðŁĶĹ": 47475, + "ðŁĶĹ": 14561, + "ðŁĶĺ": 38995, + "ðŁĶľ": 36011, + "ðŁĶĿ": 44387, + "ðŁĶĿ": 29506, + "ðŁķ": 7692, + "ðŁķº": 33958, + "ðŁķĬ": 42624, + "ðŁķĬ": 37760, + "ðŁĸ": 6269, + "ðŁĸ¤": 17603, + "ðŁĸ¤": 10860, + "ðŁĸ¥": 47990, + "ðŁĹ": 7045, + "ðŁĹ£": 33232, + "ðŁĹ£": 18583, + "ðŁĹ£ï¸ı": 37476, + "ðŁĹĵ": 34335, + "ðŁĹĵ": 28773, + "ðŁĹĵï¸ı": 39847, + "ðŁĺ": 668, + "ðŁĺ¡": 21968, + "ðŁĺ¡": 17452, + "ðŁĺ¡ðŁĺ¡": 37223, + "ðŁĺ¢": 14308, + "ðŁĺ¢": 9925, + "ðŁĺ¢ðŁĺ¢": 32923, + "ðŁĺ¢ðŁĺ¢": 47921, + "ðŁĺ£": 32718, + "ðŁĺ¤": 26872, + "ðŁĺ¤": 20740, + "ðŁĺ¥": 38383, + "ðŁĺ¥": 23951, + "ðŁĺ¨": 38080, + "ðŁĺ©": 9051, + "ðŁĺ©": 9494, + "ðŁĺ©ðŁĺ©": 22820, + "ðŁĺ©ðŁĺ©": 38031, + "ðŁĺ©ðŁĺ©ðŁĺ©": 49063, + "ðŁĺª": 38181, + "ðŁĺª": 22243, + "ðŁĺ«": 25141, + "ðŁĺ«": 22340, + "ðŁĺ¬": 23704, + "ðŁĺ¬": 14549, + "ðŁĺ®": 40163, + "ðŁĺ®": 21616, + "ðŁĺ¯": 37858, + "ðŁĺ°": 34728, + "ðŁĺ±": 10938, + "ðŁĺ±": 9055, + "ðŁĺ±ðŁĺ±": 22061, + "ðŁĺ±ðŁĺ±": 40767, + "ðŁĺ±ðŁĺ±ðŁĺ±": 40909, + "ðŁĺ²": 40460, + "ðŁĺ²": 24620, + "ðŁĺ³": 12047, + "ðŁĺ³": 8223, + "ðŁĺ³ðŁĺ³": 32592, + "ðŁĺ´": 23527, + "ðŁĺ´": 16415, + "ðŁĺ´ðŁĺ´": 49307, + "ðŁĺµ": 39368, + "ðŁĺ¶": 35207, + "ðŁĺ·": 37943, + "ðŁĺ·": 25759, + "ðŁĺ¸": 36912, + "ðŁĺ¹": 26477, + "ðŁĺ¹": 26573, + "ðŁĺ¹ðŁĺ¹": 46287, + "ðŁĺº": 40613, + "ðŁĺ»": 15453, + "ðŁĺ»": 12911, + "ðŁĺ»ðŁĺ»": 34414, + "ðŁĺ¼": 44245, + "ðŁĺ½": 45156, + "ðŁĺĢ": 12832, + "ðŁĺĢ": 7334, + "ðŁĺĢðŁĺĢ": 34503, + "ðŁĺģ": 6967, + "ðŁĺģ": 4821, + "ðŁĺģðŁĺģ": 37900, + "ðŁĺģðŁĺģ": 19213, + "ðŁĺģðŁĺģðŁĺģ": 29083, + "ðŁĺĤ": 1424, + "ðŁĺĤ)": 42643, + "ðŁĺĤ.": 42550, + "ðŁĺĤ": 1558, + "ðŁĺĤâĿ¤ï¸ı": 36412, + "ðŁĺĤðŁijĮ": 42000, + "ðŁĺĤðŁĺĤ": 2286, + "ðŁĺĤðŁĺĤ": 4112, + "ðŁĺĤðŁĺĤðŁĺĤ": 22233, + "ðŁĺĤðŁĺĤðŁĺĤ": 4887, + "ðŁĺĤðŁĺĤðŁĺĤðŁĺĤ": 9936, + "ðŁĺĤðŁĺĤðŁĺĤðŁĺĤ": 11522, + "ðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤ": 19295, + "ðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤ": 33415, + "ðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤ": 48973, + "ðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤ": 28504, + "ðŁĺĤðŁĺį": 43128, + "ðŁĺĤðŁĺŃ": 28965, + "ðŁĺĤðŁĺŃ": 25802, + "ðŁĺĥ": 14079, + "ðŁĺĥ": 8520, + "ðŁĺĥðŁĺĥ": 38358, + "ðŁĺĦ": 12141, + "ðŁĺĦ": 7624, + "ðŁĺĦðŁĺĦ": 32312, + "ðŁĺħ": 15245, + "ðŁĺħ": 9188, + "ðŁĺħðŁĺħ": 39078, + "ðŁĺĨ": 16541, + "ðŁĺĨ": 10943, + "ðŁĺĨðŁĺĨ": 39503, + "ðŁĺĩ": 21694, + "ðŁĺĩ": 13091, + "ðŁĺĪ": 14377, + "ðŁĺĪ": 9756, + "ðŁĺĪðŁĺĪ": 44473, + "ðŁĺī": 9740, + "ðŁĺī": 4955, + "ðŁĺīðŁĺī": 40430, + "ðŁĺĬ": 4692, + "ðŁĺĬ": 3020, + "ðŁĺĬâĿ¤ï¸ı": 43606, + "ðŁĺĬðŁĺĬ": 12838, + "ðŁĺĬðŁĺĬ": 20842, + "ðŁĺĬðŁĺĬðŁĺĬ": 28685, + "ðŁĺĬðŁĺĬðŁĺĬðŁĺĬ": 35519, + "ðŁĺĭ": 12391, + "ðŁĺĭ": 7203, + "ðŁĺĭðŁĺĭ": 33304, + "ðŁĺĮ": 19221, + "ðŁĺĮ": 12163, + "ðŁĺį": 1796, + "ðŁĺį#": 42357, + "ðŁĺį.": 48579, + "ðŁĺį": 1754, + "ðŁĺįâĿ¤": 29122, + "ðŁĺįâĿ¤ï¸ı": 21945, + "ðŁĺįðŁijĮ": 41005, + "ðŁĺįðŁĴķ": 35946, + "ðŁĺįðŁĶ¥": 46648, + "ðŁĺįðŁĺĤ": 48715, + "ðŁĺįðŁĺį": 3663, + "ðŁĺįðŁĺį": 6471, + "ðŁĺįðŁĺįðŁĺį": 30614, + "ðŁĺįðŁĺįðŁĺį": 7703, + "ðŁĺįðŁĺįðŁĺįðŁĺį": 16603, + "ðŁĺįðŁĺįðŁĺįðŁĺį": 18925, + "ðŁĺįðŁĺįðŁĺįðŁĺįðŁĺį": 32078, + "ðŁĺįðŁĺįðŁĺįðŁĺįðŁĺįðŁĺįðŁĺįðŁĺį": 48683, + "ðŁĺįðŁĺĺ": 29646, + "ðŁĺįðŁĺĺ": 19849, + "ðŁĺįðŁĺŃ": 39555, + "ðŁĺİ": 7426, + "ðŁĺİ": 4345, + "ðŁĺİðŁĺİ": 24048, + "ðŁĺİðŁĺİðŁĺİ": 39742, + "ðŁĺı": 11624, + "ðŁĺı": 6909, + "ðŁĺıðŁĺı": 38151, + "ðŁĺIJ": 38586, + "ðŁĺIJ": 19618, + "ðŁĺij": 32469, + "ðŁĺij": 18937, + "ðŁĺĴ": 20792, + "ðŁĺĴ": 11702, + "ðŁĺĵ": 28733, + "ðŁĺĶ": 19532, + "ðŁĺĶ": 11432, + "ðŁĺķ": 45741, + "ðŁĺķ": 20602, + "ðŁĺĸ": 35006, + "ðŁĺĺ": 4240, + "ðŁĺĺ": 3352, + "ðŁĺĺâĿ¤": 48409, + "ðŁĺĺâĿ¤ï¸ı": 39150, + "ðŁĺĺðŁĺį": 38176, + "ðŁĺĺðŁĺĺ": 15663, + "ðŁĺĺðŁĺĺ": 10507, + "ðŁĺĺðŁĺĺðŁĺĺ": 20208, + "ðŁĺĺðŁĺĺðŁĺĺðŁĺĺ": 44892, + "ðŁĺĻ": 36201, + "ðŁĺĻ": 29209, + "ðŁĺļ": 24897, + "ðŁĺļ": 19102, + "ðŁĺĽ": 24550, + "ðŁĺĽ": 15745, + "ðŁĺľ": 13226, + "ðŁĺľ": 7830, + "ðŁĺľðŁĺľ": 43065, + "ðŁĺĿ": 20064, + "ðŁĺĿ": 12970, + "ðŁĺŀ": 40458, + "ðŁĺŀ": 21103, + "ðŁĺŁ": 46947, + "ðŁĺł": 34094, + "ðŁĺŃ": 2962, + "ðŁĺŃ": 3915, + "ðŁĺŃâĿ¤ï¸ı": 29567, + "ðŁĺŃðŁĴķ": 46306, + "ðŁĺŃðŁĺĤ": 38505, + "ðŁĺŃðŁĺį": 36893, + "ðŁĺŃðŁĺŃ": 5300, + "ðŁĺŃðŁĺŃ": 11834, + "ðŁĺŃðŁĺŃðŁĺŃ": 44089, + "ðŁĺŃðŁĺŃðŁĺŃ": 13116, + "ðŁĺŃðŁĺŃðŁĺŃðŁĺŃ": 19793, + "ðŁĺŃðŁĺŃðŁĺŃðŁĺŃ": 27322, + "ðŁĺŃðŁĺŃðŁĺŃðŁĺŃðŁĺŃ": 43366, + "ðŁĻ": 1478, + "ðŁĻĢ": 43092, + "ðŁĻĤ": 32006, + "ðŁĻĤ": 14860, + "ðŁĻĥ": 27222, + "ðŁĻĥ": 15652, + "ðŁĻĦ": 20648, + "ðŁĻĦ": 13049, + "ðŁĻħ": 42702, + "ðŁĻĨ": 30050, + "ðŁĻĨ": 35730, + "ðŁĻĪ": 12661, + "ðŁĻĪ": 9516, + "ðŁĻĪðŁĻĪ": 41796, + "ðŁĻĬ": 23684, + "ðŁĻĬ": 16636, + "ðŁĻĭ": 19193, + "ðŁĻĭ": 30274, + "ðŁĻĮ": 4366, + "ðŁĻĮ": 4855, + "ðŁĻĮðŁı»": 26756, + "ðŁĻĮðŁı»": 15799, + "ðŁĻĮðŁı¼": 26584, + "ðŁĻĮðŁı¼": 15364, + "ðŁĻĮðŁı½": 36660, + "ðŁĻĮðŁı½": 22962, + "ðŁĻĮðŁı¾": 38023, + "ðŁĻĮðŁı¾": 26466, + "ðŁĻĮðŁĻĮ": 21202, + "ðŁĻĮðŁĻĮ": 30430, + "ðŁĻĮðŁĻĮðŁĻĮ": 37127, + "ðŁĻı": 4260, + "ðŁĻı": 5503, + "ðŁĻıðŁı»": 25100, + "ðŁĻıðŁı»": 16650, + "ðŁĻıðŁı¼": 31163, + "ðŁĻıðŁı¼": 18952, + "ðŁĻıðŁı½": 34103, + "ðŁĻıðŁı½": 21540, + "ðŁĻıðŁı¾": 34277, + "ðŁĻıðŁı¾": 21979, + "ðŁĻıðŁĻı": 18227, + "ðŁĻıðŁĻı": 26510, + "ðŁĻıðŁĻıðŁĻı": 31702, + "ðŁļ": 2730, + "ðŁļ¨": 12198, + "ðŁļ¨": 6056, + "ðŁļ¨ðŁļ¨": 36487, + "ðŁļ¨ðŁļ¨": 21440, + "ðŁļ¨ðŁļ¨ðŁļ¨": 41515, + "ðŁļ©": 44514, + "ðŁļ«": 35291, + "ðŁļ²": 37085, + "ðŁļ´": 30825, + "ðŁļ¶": 46060, + "ðŁļĢ": 22400, + "ðŁļĢ": 13542, + "ðŁļĢðŁļĢ": 49033, + "ðŁļĤ": 38949, + "ðŁļĮ": 46891, + "ðŁļĹ": 33054, + "ðŁļĹ": 22783, + "ðŁļĺ": 35825, + "ðŁļĻ": 48487, + "ðŁĽ": 11306, + "ñ": 173, + "ñ": 429, + "ò": 174, + "ò": 430, + "ó": 175, + "ó": 431, + "ô": 176, + "ô": 432, + "õ": 177, + "õ": 433, + "ö": 178, + "ö": 434, + "÷": 179, + "÷": 435, + "ø": 180, + "ø": 436, + "ù": 181, + "ù": 437, + "ú": 182, + "ú": 438, + "û": 183, + "û": 439, + "ü": 184, + "ü": 440, + "ý": 185, + "ý": 441, + "þ": 186, + "þ": 442, + "ÿ": 187, + "ÿ": 443, + "Ā": 188, + "Ā": 444, + "ā": 189, + "ā": 445, + "Ă": 190, + "Ă": 446, + "ă": 191, + "ă": 447, + "Ą": 192, + "Ą": 448, + "ą": 193, + "ą": 449, + "Ć": 194, + "Ć": 450, + "ć": 195, + "ć": 451, + "Ĉ": 196, + "Ĉ": 452, + "ĉ": 197, + "ĉ": 453, + "Ċ": 198, + "Ċ": 454, + "ċ": 199, + "ċ": 455, + "Č": 200, + "Č": 456, + "č": 201, + "č": 457, + "Ď": 202, + "Ď": 458, + "ď": 203, + "ď": 459, + "Đ": 204, + "Đ": 460, + "đ": 205, + "đ": 461, + "Ē": 206, + "Ē": 462, + "ē": 207, + "ē": 463, + "Ĕ": 208, + "Ĕ": 464, + "ĕ": 209, + "ĕ": 465, + "Ė": 210, + "Ė": 466, + "ė": 211, + "ė": 467, + "Ę": 212, + "Ę": 468, + "ę": 213, + "ę": 469, + "Ě": 214, + "Ě": 470, + "ě": 215, + "ě": 471, + "Ĝ": 216, + "Ĝ": 472, + "ĝ": 217, + "ĝ": 473, + "Ğ": 218, + "Ğ": 474, + "ğ": 219, + "ğ": 475, + "Ġ": 220, + "Ġ": 476, + "ġ": 221, + "ġ": 477, + "Ģ": 222, + "Ģ": 478, + "Ģï¸ı": 9668, + "Ģï¸ı": 5511, + "ģ": 223, + "ģ": 479, + "ģà¸": 15016, + "Ĥ": 224, + "Ĥ": 480, + "Ĥâĸ": 29036, + "ĤâĸĤâĸ": 30832, + "ĥ": 225, + "ĥ": 481, + "Ħ": 226, + "Ħ": 482, + "Ħà¸": 20537, + "Ħë": 34462, + "Ħëĭ": 25170, + "ħ": 227, + "ħ": 483, + "ħï¸ı": 33950, + "Ĩ": 228, + "Ĩ": 484, + "ĩ": 229, + "ĩ": 485, + "Ī": 230, + "Ī": 486, + "ī": 231, + "ī": 487, + "īï¸ı": 37463, + "Ĭ": 232, + "Ĭ": 488, + "Ĭãģ": 30294, + "ĭ": 233, + "ĭ": 489, + "ĭãģ": 36218, + "ĭãĤ": 45737, + "Į": 234, + "Į": 490, + "ĮãĤĬãģ": 45969, + "ĮãĤĬãģŁãģĦ": 47021, + "Įë": 17003, + "į": 235, + "į": 491, + "İ": 236, + "İ": 492, + "ı": 237, + "ı": 493, + "IJ": 238, + "IJ": 494, + "ij": 239, + "ij": 495, + "Ĵ": 240, + "Ĵ": 496, + "ĵ": 241, + "ĵ": 497, + "Ķ": 242, + "Ķ": 498, + "Ķë": 37978, + "Ķï¸ı": 24395, + "Ķï¸ı": 7443, + "ķ": 243, + "ķ": 499, + "ķãĤ": 26609, + "ķï¸ı": 44853, + "ĸ": 244, + "ĸ": 500, + "ĸï¸ı": 28877, + "Ĺ": 245, + "Ĺ": 501, + "ĺ": 246, + "ĺ": 502, + "Ļ": 247, + "Ļ": 503, + "ļ": 248, + "ļ": 504, + "Ľ": 249, + "Ľ": 505, + "ľ": 250, + "ľ": 506, + "ľë": 39810, + "Ŀ": 251, + "Ŀ": 507, + "ŀ": 252, + "ŀ": 508, + "Ł": 253, + "Ł": 509, + "ŁãģĦ": 46023, + "ł": 254, + "ł": 510, + "łï¸ı": 27899, + "łï¸ı": 12715, + "łĪ": 43364, + "Ń": 255, + "Ń": 511 +} diff --git a/configs/sdxl-refiner/unet/config.json b/configs/sdxl-refiner/unet/config.json new file mode 100644 index 000000000..a800deb05 --- /dev/null +++ b/configs/sdxl-refiner/unet/config.json @@ -0,0 +1,69 @@ +{ + "_class_name": "UNet2DConditionModel", + "_diffusers_version": "0.19.0.dev0", + "act_fn": "silu", + "addition_embed_type": "text_time", + "addition_embed_type_num_heads": 64, + "addition_time_embed_dim": 256, + "attention_head_dim": [ + 6, + 12, + 24, + 24 + ], + "block_out_channels": [ + 384, + 768, + 1536, + 1536 + ], + "center_input_sample": false, + "class_embed_type": null, + "class_embeddings_concat": false, + "conv_in_kernel": 3, + "conv_out_kernel": 3, + "cross_attention_dim": 1280, + "cross_attention_norm": null, + "down_block_types": [ + "DownBlock2D", + "CrossAttnDownBlock2D", + "CrossAttnDownBlock2D", + "DownBlock2D" + ], + "downsample_padding": 1, + "dual_cross_attention": false, + "encoder_hid_dim": null, + "encoder_hid_dim_type": null, + "flip_sin_to_cos": true, + "freq_shift": 0, + "in_channels": 4, + "layers_per_block": 2, + "mid_block_only_cross_attention": null, + "mid_block_scale_factor": 1, + "mid_block_type": "UNetMidBlock2DCrossAttn", + "norm_eps": 1e-05, + "norm_num_groups": 32, + "num_attention_heads": null, + "num_class_embeds": null, + "only_cross_attention": false, + "out_channels": 4, + "projection_class_embeddings_input_dim": 2560, + "resnet_out_scale_factor": 1.0, + "resnet_skip_time_act": false, + "resnet_time_scale_shift": "default", + "sample_size": 128, + "time_cond_proj_dim": null, + "time_embedding_act_fn": null, + "time_embedding_dim": null, + "time_embedding_type": "positional", + "timestep_post_act": null, + "transformer_layers_per_block": 4, + "up_block_types": [ + "UpBlock2D", + "CrossAttnUpBlock2D", + "CrossAttnUpBlock2D", + "UpBlock2D" + ], + "upcast_attention": null, + "use_linear_projection": true +} diff --git a/configs/sdxl-refiner/vae/config.json b/configs/sdxl-refiner/vae/config.json new file mode 100644 index 000000000..a66a171ba --- /dev/null +++ b/configs/sdxl-refiner/vae/config.json @@ -0,0 +1,32 @@ +{ + "_class_name": "AutoencoderKL", + "_diffusers_version": "0.20.0.dev0", + "_name_or_path": "../sdxl-vae/", + "act_fn": "silu", + "block_out_channels": [ + 128, + 256, + 512, + 512 + ], + "down_block_types": [ + "DownEncoderBlock2D", + "DownEncoderBlock2D", + "DownEncoderBlock2D", + "DownEncoderBlock2D" + ], + "force_upcast": true, + "in_channels": 3, + "latent_channels": 4, + "layers_per_block": 2, + "norm_num_groups": 32, + "out_channels": 3, + "sample_size": 1024, + "scaling_factor": 0.13025, + "up_block_types": [ + "UpDecoderBlock2D", + "UpDecoderBlock2D", + "UpDecoderBlock2D", + "UpDecoderBlock2D" + ] +} diff --git a/modules/processing_args.py b/modules/processing_args.py index 054211517..4b1765e85 100644 --- a/modules/processing_args.py +++ b/modules/processing_args.py @@ -6,6 +6,7 @@ import inspect import torch import numpy as np +from PIL import Image from modules import shared, errors, sd_models, processing, processing_vae, processing_helpers, sd_hijack_hypertile, prompt_parser_diffusers, timer, extra_networks from modules.processing_callbacks import diffusers_callback_legacy, diffusers_callback, set_callbacks_p from modules.processing_helpers import resize_hires, fix_prompts, calculate_base_steps, calculate_hires_steps, calculate_refiner_steps, get_generator, set_latents, apply_circular # pylint: disable=unused-import @@ -22,7 +23,8 @@ def task_specific_kwargs(p, model): if len(getattr(p, 'init_images', [])) > 0: if isinstance(p.init_images[0], str): p.init_images = [helpers.decode_base64_to_image(i, quiet=True) for i in p.init_images] - p.init_images = [i.convert('RGB') if i.mode != 'RGB' else i for i in p.init_images if i is not None] + if isinstance(p.init_images[0], Image.Image): + p.init_images = [i.convert('RGB') if i.mode != 'RGB' else i for i in p.init_images if i is not None] if (sd_models.get_diffusers_task(model) == sd_models.DiffusersTaskType.TEXT_2_IMAGE or len(getattr(p, 'init_images', [])) == 0) and not is_img2img_model: p.ops.append('txt2img') if hasattr(p, 'width') and hasattr(p, 'height'): @@ -99,7 +101,7 @@ def task_specific_kwargs(p, model): return task_args -def set_pipeline_args(p, model, prompts: list, negative_prompts: list, prompts_2: typing.Optional[list]=None, negative_prompts_2: typing.Optional[list]=None, desc:str='', **kwargs): +def set_pipeline_args(p, model, prompts:list, negative_prompts:list, prompts_2:typing.Optional[list]=None, negative_prompts_2:typing.Optional[list]=None, prompt_attention:typing.Optional[str]=None, desc:typing.Optional[str]='', **kwargs): t0 = time.time() shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) apply_circular(p.tiling, model) @@ -118,7 +120,8 @@ def set_pipeline_args(p, model, prompts: list, negative_prompts: list, prompts_2 clip_skip = kwargs.pop("clip_skip", 1) parser = 'fixed' - if shared.opts.prompt_attention != 'fixed' and 'Onnx' not in model.__class__.__name__ and ( + prompt_attention = prompt_attention or shared.opts.prompt_attention + if prompt_attention != 'fixed' and 'Onnx' not in model.__class__.__name__ and ( 'StableDiffusion' in model.__class__.__name__ or 'StableCascade' in model.__class__.__name__ or 'Flux' in model.__class__.__name__ @@ -265,7 +268,7 @@ def set_pipeline_args(p, model, prompts: list, negative_prompts: list, prompts_2 elif 'callback' in possible: args['callback'] = diffusers_callback_legacy - if 'image' in kwargs and len(getattr(p, 'init_images', [])) == 0: + if 'image' in kwargs: p.init_images = kwargs['image'] if isinstance(kwargs['image'], list) else [kwargs['image']] # handle remaining args diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 41d412a68..c9894739d 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -199,8 +199,6 @@ def process_hires(p: processing.StableDiffusionProcessing, output): if p.is_control and hasattr(p, 'task_args') and p.task_args.get('image', None) is not None: if hasattr(shared.sd_model, "vae") and output.images is not None and len(output.images) > 0: output.images = processing_vae.vae_decode(latents=output.images, model=shared.sd_model, full_quality=p.full_quality, output_type='pil', width=p.hr_upscale_to_x, height=p.hr_upscale_to_y) # controlnet cannnot deal with latent input - p.init_images = output.images # replace so hires uses new output - # p.task_args['image'] = output.images # replace so hires uses new output update_sampler(p, shared.sd_model, second_pass=True) orig_denoise = p.denoising_strength p.denoising_strength = strength @@ -289,10 +287,8 @@ def process_refine(p: processing.StableDiffusionProcessing, output): image = processing_vae.vae_decode(latents=image, model=shared.sd_model, full_quality=p.full_quality, output_type='pil', width=p.width, height=p.height) p.extra_generation_params['Noise level'] = noise_level output_type = 'np' - if p.task_args.get('image', None) is not None and output is not None: # replace input with output so it can be used by hires/refine - # p.task_args['image'] = image - p.init_images = [image] update_sampler(p, shared.sd_refiner, second_pass=True) + shared.opts.prompt_attention = 'fixed' refiner_args = set_pipeline_args( p=p, model=shared.sd_refiner, @@ -309,6 +305,7 @@ def process_refine(p: processing.StableDiffusionProcessing, output): image=image, output_type=output_type, clip_skip=p.clip_skip, + prompt_attention='fixed', desc='Refiner', ) shared.state.sampling_steps = refiner_args.get('prior_num_inference_steps', None) or p.steps or refiner_args.get('num_inference_steps', None) diff --git a/modules/sd_detect.py b/modules/sd_detect.py index fe3d325c9..15b22c69c 100644 --- a/modules/sd_detect.py +++ b/modules/sd_detect.py @@ -170,6 +170,8 @@ def get_load_config(model_file, model_type, config_type='yaml'): return 'configs/sd15' if model_type == 'Stable Diffusion XL': return 'configs/sdxl' + if model_type == 'Stable Diffusion XL Refiner': + return 'configs/sdxl-refiner' if model_type == 'Stable Diffusion 3': return 'configs/sd3' if model_type == 'FLUX': diff --git a/modules/shared_items.py b/modules/shared_items.py index 9abb64718..17b7ce1ee 100644 --- a/modules/shared_items.py +++ b/modules/shared_items.py @@ -68,7 +68,7 @@ def get_pipelines(): 'Stable Diffusion Instruct': getattr(diffusers, 'StableDiffusionInstructPix2PixPipeline', None), 'Stable Diffusion Upscale': getattr(diffusers, 'StableDiffusionUpscalePipeline', None), 'Stable Diffusion XL': getattr(diffusers, 'StableDiffusionXLPipeline', None), - 'Stable Diffusion XL Refiner': getattr(diffusers, 'StableDiffusionXLPipeline', None), + 'Stable Diffusion XL Refiner': getattr(diffusers, 'StableDiffusionXLImg2ImgPipeline', None), 'Stable Diffusion XL Img2Img': getattr(diffusers, 'StableDiffusionXLImg2ImgPipeline', None), 'Stable Diffusion XL Inpaint': getattr(diffusers, 'StableDiffusionXLInpaintPipeline', None), 'Stable Diffusion XL Instruct': getattr(diffusers, 'StableDiffusionXLInstructPix2PixPipeline', None), diff --git a/modules/textual_inversion/textual_inversion.py b/modules/textual_inversion/textual_inversion.py index de12021a5..86c7cd260 100644 --- a/modules/textual_inversion/textual_inversion.py +++ b/modules/textual_inversion/textual_inversion.py @@ -418,7 +418,7 @@ def load_textual_inversion_embeddings(self, force_reload=False): self.word_embeddings.update(sorted_word_embeddings) displayed_embeddings = (tuple(self.word_embeddings.keys()), tuple(self.skipped_embeddings.keys())) - if self.previously_displayed_embeddings != displayed_embeddings: + if self.previously_displayed_embeddings != displayed_embeddings and shared.opts.diffusers_enable_embed: self.previously_displayed_embeddings = displayed_embeddings t1 = time.time() shared.log.info(f"Load network: type=embeddings loaded={len(self.word_embeddings)} skipped={len(self.skipped_embeddings)} time={t1-t0:.2f}") From 4f79f86f0823feb2ec5e7b56843eecc5ba4c20e7 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 30 Dec 2024 09:21:02 -0500 Subject: [PATCH 178/249] update live preview Signed-off-by: Vladimir Mandic --- modules/progress.py | 2 +- modules/shared_state.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/progress.py b/modules/progress.py index 68102b915..8e01c02b3 100644 --- a/modules/progress.py +++ b/modules/progress.py @@ -78,7 +78,7 @@ def progressapi(req: ProgressRequest): id_live_preview = req.id_live_preview live_preview = None updated = shared.state.set_current_image() - debug_log(f'Preview: job={shared.state.job} active={active} progress={current}/{total} request={id_live_preview} last={shared.state.id_live_preview} enabled={shared.opts.live_previews_enable} updated={updated} image={shared.state.current_image} elapsed={elapsed:.3f}') + debug_log(f'Preview: job={shared.state.job} active={active} progress={current}/{total} step={shared.state.current_image_sampling_step}/{shared.state.sampling_step} request={id_live_preview} last={shared.state.id_live_preview} enabled={shared.opts.live_previews_enable} busy={shared.state.preview_busy} updated={updated} image={shared.state.current_image} elapsed={elapsed:.3f}') if not active: return InternalProgressResponse(job=shared.state.job, active=active, queued=queued, paused=paused, completed=completed, id_live_preview=-1, debug=debug, textinfo="Queued..." if queued else "Waiting...") if shared.opts.live_previews_enable and (shared.state.id_live_preview != id_live_preview) and (shared.state.current_image is not None): diff --git a/modules/shared_state.py b/modules/shared_state.py index 3fe0279f1..6b4ce3da8 100644 --- a/modules/shared_state.py +++ b/modules/shared_state.py @@ -123,7 +123,7 @@ def begin(self, title="", api=None): self.skipped = False self.textinfo = None self.prediction_type = "epsilon" - self.api = api if api is not None else self.api + self.api = api or self.api self.time_start = time.time() if self.debug_output: log.debug(f'State begin: {self.job}') @@ -144,11 +144,11 @@ def end(self, api=None): self.paused = False self.interrupted = False self.skipped = False - self.api = api if api is not None else self.api + self.api = api or self.api modules.devices.torch_gc() def set_current_image(self): - if self.job == 'VAE': # avoid generating preview while vae is running + if self.job == 'VAE' or self.job == 'Upscale': # avoid generating preview while vae is running return False from modules.shared import opts, cmd_opts if cmd_opts.lowvram or self.api or (not opts.live_previews_enable) or (opts.show_progress_every_n_steps <= 0): From ca5c939b32ef7b7afdcab3165bb940c0f42d9a29 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 30 Dec 2024 10:03:34 -0500 Subject: [PATCH 179/249] fix kandinsky Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 1 + modules/processing_args.py | 6 ++++-- modules/processing_diffusers.py | 4 ++-- modules/sd_models.py | 2 ++ 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 621f65daf..9d07d45ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ - remove concurrent preview requests - hires batch - sdxl refiner + - kandinsky ## Update for 2024-12-24 diff --git a/modules/processing_args.py b/modules/processing_args.py index 4b1765e85..244948530 100644 --- a/modules/processing_args.py +++ b/modules/processing_args.py @@ -233,9 +233,11 @@ def set_pipeline_args(p, model, prompts:list, negative_prompts:list, prompts_2:t args['latents'] = p.init_latent if 'output_type' in possible: if not hasattr(model, 'vae'): - args['output_type'] = 'np' # only set latent if model has vae + kwargs['output_type'] = 'np' # only set latent if model has vae - # stable cascade + # model specific + if 'Kandinsky' in model.__class__.__name__: + kwargs['output_type'] = 'np' # only set latent if model has vae if 'StableCascade' in model.__class__.__name__: kwargs.pop("guidance_scale") # remove kwargs.pop("num_inference_steps") # remove diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index c9894739d..4f8125d4f 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -194,7 +194,7 @@ def process_hires(p: processing.StableDiffusionProcessing, output): if p.hr_force: shared.state.job_count = 2 * p.n_iter shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE) - if 'Upscale' in shared.sd_model.__class__.__name__ or 'Flux' in shared.sd_model.__class__.__name__: + if 'Upscale' in shared.sd_model.__class__.__name__ or 'Flux' in shared.sd_model.__class__.__name__ or 'Kandinsky' in shared.sd_model.__class__.__name__: output.images = processing_vae.vae_decode(latents=output.images, model=shared.sd_model, full_quality=p.full_quality, output_type='pil', width=p.width, height=p.height) if p.is_control and hasattr(p, 'task_args') and p.task_args.get('image', None) is not None: if hasattr(shared.sd_model, "vae") and output.images is not None and len(output.images) > 0: @@ -283,7 +283,7 @@ def process_refine(p: processing.StableDiffusionProcessing, output): image = output.images[i] noise_level = round(350 * p.denoising_strength) output_type='latent' - if 'Upscale' in shared.sd_refiner.__class__.__name__ or 'Flux' in shared.sd_refiner.__class__.__name__: + if 'Upscale' in shared.sd_refiner.__class__.__name__ or 'Flux' in shared.sd_refiner.__class__.__name__ or 'Kandinsky' in shared.sd_refiner.__class__.__name__: image = processing_vae.vae_decode(latents=image, model=shared.sd_model, full_quality=p.full_quality, output_type='pil', width=p.width, height=p.height) p.extra_generation_params['Noise level'] = noise_level output_type = 'np' diff --git a/modules/sd_models.py b/modules/sd_models.py index 99e821667..a3b6874c8 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -1325,6 +1325,8 @@ def set_attn(pipe, attention): module.set_attn_processor(p.HunyuanAttnProcessor2_0()) elif module.__class__.__name__ in ['AuraFlowTransformer2DModel']: module.set_attn_processor(p.AuraFlowAttnProcessor2_0()) + elif 'KandinskyCombinedPipeline' in pipe.__class__.__name__: + pass elif 'Transformer' in module.__class__.__name__: pass # unknown transformer so probably dont want to force attention processor else: From a2e1ae48c9d365210cc87712eeb9e2f62e25f1df Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 30 Dec 2024 10:17:00 -0500 Subject: [PATCH 180/249] update live preview Signed-off-by: Vladimir Mandic --- modules/shared_state.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/shared_state.py b/modules/shared_state.py index 6b4ce3da8..472e88856 100644 --- a/modules/shared_state.py +++ b/modules/shared_state.py @@ -113,6 +113,7 @@ def begin(self, title="", api=None): self.current_sigma_next = None self.id_live_preview = 0 self.interrupted = False + self.preview_busy = False self.job = title self.job_count = -1 self.frame_count = -1 @@ -136,11 +137,14 @@ def end(self, api=None): # log.debug(f'Access state.end: {fn}') # pylint: disable=protected-access self.time_start = time.time() if self.debug_output: + if self.preview_busy: + log.debug('State end: preview busy') log.debug(f'State end: {self.job} time={time.time() - self.time_start:.2f}') self.job = "" self.job_count = 0 self.job_no = 0 self.frame_count = 0 + self.preview_busy = False self.paused = False self.interrupted = False self.skipped = False From 5749b9fc80af9f6ffe1b9aed0726b70e32a0bd7d Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 30 Dec 2024 11:18:17 -0500 Subject: [PATCH 181/249] reenable preview sigma calculations Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 4 +++- modules/shared_state.py | 18 +++++++++--------- scripts/xyz_grid_on.py | 1 + 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d07d45ff..6e9a5c251 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Change Log for SD.Next -## Update for 2024-12-29 +## Update for 2024-12-30 - **LoRA**: - **Sana** support @@ -11,6 +11,7 @@ - **LTXVideo** optimizations: full offload, quantization and tiling support - VAE tiling granular options in *settings -> variable auto encoder* - UI: live preview optimizations and error handling +- UI: live preview sigma calulations, thanks @Disty0 - UI: CSS optimizations when log view is disabled - Samplers: add flow shift options and separate dynamic thresholding from dynamic shifting - **Fixes** @@ -20,6 +21,7 @@ - interrogate caption with T5 - on-the-fly quantization using TorchAO - remove concurrent preview requests + - xyz grid recover on error - hires batch - sdxl refiner - kandinsky diff --git a/modules/shared_state.py b/modules/shared_state.py index 472e88856..dd3905de9 100644 --- a/modules/shared_state.py +++ b/modules/shared_state.py @@ -169,15 +169,15 @@ def do_set_current_image(self): try: sample = self.current_latent self.current_image_sampling_step = self.sampling_step - """ - if self.current_noise_pred is not None and self.current_sigma is not None and self.current_sigma_next is not None: - original_sample = sample - (self.current_noise_pred * (self.current_sigma_next-self.current_sigma)) - # RuntimeError: The size of tensor a (128) must match the size of tensor b (64) at non-singleton dimension 3 - if self.prediction_type in {"epsilon", "flow_prediction"}: - sample = original_sample - (self.current_noise_pred * self.current_sigma) - elif self.prediction_type == "v_prediction": - sample = self.current_noise_pred * (-self.current_sigma / (self.current_sigma**2 + 1) ** 0.5) + (original_sample / (self.current_sigma**2 + 1)) # pylint: disable=invalid-unary-operand-type - """ + try: + if self.current_noise_pred is not None and self.current_sigma is not None and self.current_sigma_next is not None: + original_sample = sample - (self.current_noise_pred * (self.current_sigma_next-self.current_sigma)) + if self.prediction_type in {"epsilon", "flow_prediction"}: + sample = original_sample - (self.current_noise_pred * self.current_sigma) + elif self.prediction_type == "v_prediction": + sample = self.current_noise_pred * (-self.current_sigma / (self.current_sigma**2 + 1) ** 0.5) + (original_sample / (self.current_sigma**2 + 1)) # pylint: disable=invalid-unary-operand-type + except Exception: + pass # ignore sigma errors image = sd_samplers.samples_to_image_grid(sample) if shared.opts.show_progress_grid else sd_samplers.sample_to_image(sample) self.assign_current_image(image) self.preview_busy = False diff --git a/scripts/xyz_grid_on.py b/scripts/xyz_grid_on.py index aa0897442..ba23356da 100644 --- a/scripts/xyz_grid_on.py +++ b/scripts/xyz_grid_on.py @@ -379,6 +379,7 @@ def cell(x, y, z, ix, iy, iz): ) if not processed.images: + active = False return processed # something broke, no further handling needed. # processed.images = (1)*grid + (z > 1 ? z : 0)*subgrids + (x*y*z)*images have_grid = 1 if include_grid else 0 From 7609e5c204f680b9629cf09b59855fa268d656af Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 30 Dec 2024 11:42:04 -0500 Subject: [PATCH 182/249] remove all ldm imports when running in native mode Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 1 + modules/hypernetworks/hypernetwork.py | 2 +- modules/postprocess/ldsr_model.py | 15 +++--- modules/sd_hijack.py | 66 +++++++++++++-------------- modules/sd_hijack_optimizations.py | 7 ++- modules/sd_hijack_unet.py | 31 +++++++------ modules/sd_models.py | 2 +- webui.py | 2 +- 8 files changed, 65 insertions(+), 61 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e9a5c251..5e11c04c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ - UI: live preview sigma calulations, thanks @Disty0 - UI: CSS optimizations when log view is disabled - Samplers: add flow shift options and separate dynamic thresholding from dynamic shifting +- Refactor: remove all LDM imports if running in native mode - **Fixes** - do not show disabled networks - image width/height calculation when doing img2img diff --git a/modules/hypernetworks/hypernetwork.py b/modules/hypernetworks/hypernetwork.py index 1bb0c8ade..e15cf7724 100644 --- a/modules/hypernetworks/hypernetwork.py +++ b/modules/hypernetworks/hypernetwork.py @@ -6,7 +6,6 @@ from torch import einsum from torch.nn.init import normal_, xavier_normal_, xavier_uniform_, kaiming_normal_, kaiming_uniform_, zeros_ from einops import rearrange, repeat -from ldm.util import default from modules import devices, shared, hashes, errors, files_cache @@ -327,6 +326,7 @@ def apply_hypernetworks(hypernetworks, context, layer=None): def attention_CrossAttention_forward(self, x, context=None, mask=None): + from ldm.util import default h = self.heads q = self.to_q(x) context = default(context, x) diff --git a/modules/postprocess/ldsr_model.py b/modules/postprocess/ldsr_model.py index f9ddb357a..8abb7081a 100644 --- a/modules/postprocess/ldsr_model.py +++ b/modules/postprocess/ldsr_model.py @@ -1,15 +1,16 @@ import os import sys import traceback - from modules.upscaler import Upscaler, UpscalerData -from modules.ldsr.ldsr_model_arch import LDSR from modules import shared, script_callbacks -import modules.ldsr.sd_hijack_autoencoder # pylint: disable=unused-import -import modules.ldsr.sd_hijack_ddpm_v1 # pylint: disable=unused-import -class UpscalerLDSR(Upscaler): +class Dummy: + pass + +cls = Upscaler if not shared.native else Dummy + +class UpscalerLDSR(cls): def __init__(self, user_path): self.name = "LDSR" self.user_path = user_path @@ -20,6 +21,9 @@ def __init__(self, user_path): self.scalers = [scaler_data] def load_model(self, path: str): + from modules.ldsr.ldsr_model_arch import LDSR + import modules.ldsr.sd_hijack_autoencoder # pylint: disable=unused-import + import modules.ldsr.sd_hijack_ddpm_v1 # pylint: disable=unused-import # Remove incorrect project.yaml file if too big yaml_path = os.path.join(self.model_path, "project.yaml") old_model_path = os.path.join(self.model_path, "model.pth") @@ -50,7 +54,6 @@ def load_model(self, path: str): try: return LDSR(model, yaml) - except Exception: print("Error importing LDSR:", file=sys.stderr) print(traceback.format_exc(), file=sys.stderr) diff --git a/modules/sd_hijack.py b/modules/sd_hijack.py index 75e51f4da..8350006b8 100644 --- a/modules/sd_hijack.py +++ b/modules/sd_hijack.py @@ -7,34 +7,30 @@ import diffusers from modules import shared -shared.log.debug('Importing LDM') -stdout = io.StringIO() -with contextlib.redirect_stdout(stdout): - import ldm.modules.attention - import ldm.modules.distributions.distributions - import ldm.modules.diffusionmodules.model - import ldm.modules.diffusionmodules.openaimodel - import ldm.models.diffusion.ddim - import ldm.models.diffusion.plms - import ldm.modules.encoders.modules - -import modules.textual_inversion.textual_inversion -from modules import devices, sd_hijack_optimizations +if not shared.native: + shared.log.warning('Importing LDM') + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + import ldm.modules.attention + import ldm.modules.distributions.distributions + import ldm.modules.diffusionmodules.model + import ldm.modules.diffusionmodules.openaimodel + import ldm.models.diffusion.ddim + import ldm.models.diffusion.plms + import ldm.modules.encoders.modules + attention_CrossAttention_forward = ldm.modules.attention.CrossAttention.forward + diffusionmodules_model_nonlinearity = ldm.modules.diffusionmodules.model.nonlinearity + diffusionmodules_model_AttnBlock_forward = ldm.modules.diffusionmodules.model.AttnBlock.forward + ldm.modules.attention.MemoryEfficientCrossAttention = ldm.modules.attention.CrossAttention + ldm.modules.attention.BasicTransformerBlock.ATTENTION_MODES["softmax-xformers"] = ldm.modules.attention.CrossAttention + # silence new console spam from SD2 + ldm.modules.attention.print = lambda *args: None + ldm.modules.diffusionmodules.model.print = lambda *args: None + +from modules import devices, sd_hijack_optimizations # pylint: disable=ungrouped-imports +from modules.textual_inversion import textual_inversion from modules.hypernetworks import hypernetwork -attention_CrossAttention_forward = ldm.modules.attention.CrossAttention.forward -diffusionmodules_model_nonlinearity = ldm.modules.diffusionmodules.model.nonlinearity -diffusionmodules_model_AttnBlock_forward = ldm.modules.diffusionmodules.model.AttnBlock.forward - -# new memory efficient cross attention blocks do not support hypernets and we already -# have memory efficient cross attention anyway, so this disables SD2.0's memory efficient cross attention -ldm.modules.attention.MemoryEfficientCrossAttention = ldm.modules.attention.CrossAttention -ldm.modules.attention.BasicTransformerBlock.ATTENTION_MODES["softmax-xformers"] = ldm.modules.attention.CrossAttention - -# silence new console spam from SD2 -ldm.modules.attention.print = lambda *args: None -ldm.modules.diffusionmodules.model.print = lambda *args: None - current_optimizer = SimpleNamespace(**{ "name": "none" }) def apply_optimizations(): @@ -86,9 +82,10 @@ def apply_optimizations(): def undo_optimizations(): - ldm.modules.attention.CrossAttention.forward = hypernetwork.attention_CrossAttention_forward - ldm.modules.diffusionmodules.model.nonlinearity = diffusionmodules_model_nonlinearity - ldm.modules.diffusionmodules.model.AttnBlock.forward = diffusionmodules_model_AttnBlock_forward + if not shared.native: + ldm.modules.attention.CrossAttention.forward = hypernetwork.attention_CrossAttention_forward + ldm.modules.diffusionmodules.model.nonlinearity = diffusionmodules_model_nonlinearity + ldm.modules.diffusionmodules.model.AttnBlock.forward = diffusionmodules_model_AttnBlock_forward def fix_checkpoint(): @@ -153,7 +150,7 @@ class StableDiffusionModelHijack: clip = None optimization_method = None - embedding_db = modules.textual_inversion.textual_inversion.EmbeddingDatabase() + embedding_db = textual_inversion.EmbeddingDatabase() def __init__(self): self.embedding_db.add_embedding_dir(shared.opts.embeddings_dir) @@ -330,11 +327,10 @@ def register_buffer(self, name, attr): setattr(self, name, attr) -ldm.models.diffusion.ddim.DDIMSampler.register_buffer = register_buffer -ldm.models.diffusion.plms.PLMSSampler.register_buffer = register_buffer - -# Ensure samping from Guassian for DDPM follows types -ldm.modules.distributions.distributions.DiagonalGaussianDistribution.sample = lambda self: self.mean.to(self.parameters.dtype) + self.std.to(self.parameters.dtype) * torch.randn(self.mean.shape, dtype=self.parameters.dtype).to(device=self.parameters.device) +if not shared.native: + ldm.models.diffusion.ddim.DDIMSampler.register_buffer = register_buffer + ldm.models.diffusion.plms.PLMSSampler.register_buffer = register_buffer + ldm.modules.distributions.distributions.DiagonalGaussianDistribution.sample = lambda self: self.mean.to(self.parameters.dtype) + self.std.to(self.parameters.dtype) * torch.randn(self.mean.shape, dtype=self.parameters.dtype).to(device=self.parameters.device) # Upcast BF16 to FP32 diff --git a/modules/sd_hijack_optimizations.py b/modules/sd_hijack_optimizations.py index 6421de0de..31089551e 100644 --- a/modules/sd_hijack_optimizations.py +++ b/modules/sd_hijack_optimizations.py @@ -4,13 +4,16 @@ import psutil import torch from torch import einsum -from ldm.util import default from einops import rearrange from modules import shared, errors, devices from modules.hypernetworks import hypernetwork from .sub_quadratic_attention import efficient_dot_product_attention # pylint: disable=relative-beyond-top-level +if not shared.native: + from ldm.util import default + + if shared.opts.cross_attention_optimization == "xFormers": try: import xformers.ops # pylint: disable=import-error @@ -47,7 +50,7 @@ def split_cross_attention_forward_v1(self, x, context=None, mask=None): # pylint h = self.heads q_in = self.to_q(x) - context = default(context, x) + context = default(context, x) # pylint: disable=possibly-used-before-assignment context_k, context_v = hypernetwork.apply_hypernetworks(shared.loaded_hypernetworks, context) k_in = self.to_k(context_k) diff --git a/modules/sd_hijack_unet.py b/modules/sd_hijack_unet.py index 66040c5aa..ce0b0c983 100644 --- a/modules/sd_hijack_unet.py +++ b/modules/sd_hijack_unet.py @@ -1,7 +1,7 @@ import torch from packaging import version -from modules import devices +from modules import devices, shared from modules.sd_hijack_utils import CondFunc @@ -59,21 +59,22 @@ def forward(self, input): # pylint: disable=redefined-builtin def hijack_ddpm_edit(): global ddpm_edit_hijack # pylint: disable=global-statement if not ddpm_edit_hijack: - CondFunc('modules.hijack.ddpm_edit.LatentDiffusion.decode_first_stage', first_stage_sub, first_stage_cond) - CondFunc('modules.hijack.ddpm_edit.LatentDiffusion.encode_first_stage', first_stage_sub, first_stage_cond) + CondFunc('modules.hijack.ddpm_edit.LatentDiffusion.decode_first_stage', first_stage_sub, first_stage_cond) # pylint: disable=possibly-used-before-assignment + CondFunc('modules.hijack.ddpm_edit.LatentDiffusion.encode_first_stage', first_stage_sub, first_stage_cond) # pylint: disable=possibly-used-before-assignment ddpm_edit_hijack = CondFunc('modules.hijack.ddpm_edit.LatentDiffusion.apply_model', apply_model, unet_needs_upcast) unet_needs_upcast = lambda *args, **kwargs: devices.unet_needs_upcast # pylint: disable=unnecessary-lambda-assignment -CondFunc('ldm.models.diffusion.ddpm.LatentDiffusion.apply_model', apply_model, unet_needs_upcast) -CondFunc('ldm.modules.diffusionmodules.openaimodel.timestep_embedding', lambda orig_func, timesteps, *args, **kwargs: orig_func(timesteps, *args, **kwargs).to(torch.float32 if timesteps.dtype == torch.int64 else devices.dtype_unet), unet_needs_upcast) -if version.parse(torch.__version__) <= version.parse("1.13.2") or torch.cuda.is_available(): - CondFunc('ldm.modules.diffusionmodules.util.GroupNorm32.forward', lambda orig_func, self, *args, **kwargs: orig_func(self.float(), *args, **kwargs), unet_needs_upcast) - CondFunc('ldm.modules.attention.GEGLU.forward', lambda orig_func, self, x: orig_func(self.float(), x.float()).to(devices.dtype_unet), unet_needs_upcast) - CondFunc('open_clip.transformer.ResidualAttentionBlock.__init__', lambda orig_func, *args, **kwargs: (kwargs.update({'act_layer': GELUHijack}) and False) or orig_func(*args, **kwargs), lambda _, *args, **kwargs: kwargs.get('act_layer') is None or kwargs['act_layer'] == torch.nn.GELU) - -first_stage_cond = lambda _, self, *args, **kwargs: devices.unet_needs_upcast and self.model.diffusion_model.dtype == torch.float16 # pylint: disable=unnecessary-lambda-assignment -first_stage_sub = lambda orig_func, self, x, **kwargs: orig_func(self, x.to(devices.dtype_vae), **kwargs) # pylint: disable=unnecessary-lambda-assignment -CondFunc('ldm.models.diffusion.ddpm.LatentDiffusion.decode_first_stage', first_stage_sub, first_stage_cond) -CondFunc('ldm.models.diffusion.ddpm.LatentDiffusion.encode_first_stage', first_stage_sub, first_stage_cond) -CondFunc('ldm.models.diffusion.ddpm.LatentDiffusion.get_first_stage_encoding', lambda orig_func, *args, **kwargs: orig_func(*args, **kwargs).float(), first_stage_cond) +if not shared.native: + CondFunc('ldm.models.diffusion.ddpm.LatentDiffusion.apply_model', apply_model, unet_needs_upcast) + CondFunc('ldm.modules.diffusionmodules.openaimodel.timestep_embedding', lambda orig_func, timesteps, *args, **kwargs: orig_func(timesteps, *args, **kwargs).to(torch.float32 if timesteps.dtype == torch.int64 else devices.dtype_unet), unet_needs_upcast) + if version.parse(torch.__version__) <= version.parse("1.13.2") or torch.cuda.is_available(): + CondFunc('ldm.modules.diffusionmodules.util.GroupNorm32.forward', lambda orig_func, self, *args, **kwargs: orig_func(self.float(), *args, **kwargs), unet_needs_upcast) + CondFunc('ldm.modules.attention.GEGLU.forward', lambda orig_func, self, x: orig_func(self.float(), x.float()).to(devices.dtype_unet), unet_needs_upcast) + CondFunc('open_clip.transformer.ResidualAttentionBlock.__init__', lambda orig_func, *args, **kwargs: (kwargs.update({'act_layer': GELUHijack}) and False) or orig_func(*args, **kwargs), lambda _, *args, **kwargs: kwargs.get('act_layer') is None or kwargs['act_layer'] == torch.nn.GELU) + + first_stage_cond = lambda _, self, *args, **kwargs: devices.unet_needs_upcast and self.model.diffusion_model.dtype == torch.float16 # pylint: disable=unnecessary-lambda-assignment + first_stage_sub = lambda orig_func, self, x, **kwargs: orig_func(self, x.to(devices.dtype_vae), **kwargs) # pylint: disable=unnecessary-lambda-assignment + CondFunc('ldm.models.diffusion.ddpm.LatentDiffusion.decode_first_stage', first_stage_sub, first_stage_cond) + CondFunc('ldm.models.diffusion.ddpm.LatentDiffusion.encode_first_stage', first_stage_sub, first_stage_cond) + CondFunc('ldm.models.diffusion.ddpm.LatentDiffusion.get_first_stage_encoding', lambda orig_func, *args, **kwargs: orig_func(*args, **kwargs).float(), first_stage_cond) diff --git a/modules/sd_models.py b/modules/sd_models.py index a3b6874c8..f962eeaca 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -15,7 +15,6 @@ import safetensors.torch import accelerate from omegaconf import OmegaConf -from ldm.util import instantiate_from_config from modules import paths, shared, shared_state, modelloader, devices, script_callbacks, sd_vae, sd_unet, errors, sd_models_config, sd_models_compile, sd_hijack_accelerate, sd_detect from modules.timer import Timer, process as process_timer from modules.memstats import memory_stats @@ -1382,6 +1381,7 @@ def get_native(pipe: diffusers.DiffusionPipeline): def load_model(checkpoint_info=None, already_loaded_state_dict=None, timer=None, op='model'): + from ldm.util import instantiate_from_config from modules import lowvram, sd_hijack checkpoint_info = checkpoint_info or select_checkpoint(op=op) if checkpoint_info is None: diff --git a/webui.py b/webui.py index 85725efe4..d3a0328eb 100644 --- a/webui.py +++ b/webui.py @@ -13,7 +13,7 @@ import torch # pylint: disable=wrong-import-order from modules import timer, errors, paths # pylint: disable=unused-import from installer import log, git_commit, custom_excepthook -import ldm.modules.encoders.modules # pylint: disable=unused-import, wrong-import-order +# import ldm.modules.encoders.modules # pylint: disable=unused-import, wrong-import-order from modules import shared, extensions, gr_tempdir, modelloader # pylint: disable=ungrouped-imports from modules import extra_networks, ui_extra_networks # pylint: disable=ungrouped-imports from modules.paths import create_paths From e7f173f4f9759f88001145cde4a8c178da3dd1b5 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 30 Dec 2024 12:21:47 -0500 Subject: [PATCH 183/249] samplers autodetect if sigma is required and available Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 16 ++++++++++------ modules/sd_samplers.py | 12 ++++++++---- modules/sd_samplers_diffusers.py | 10 +++++++++- 3 files changed, 27 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e11c04c5..06524e1c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,12 +9,16 @@ - add legacy option in *settings -> networks* - **HunyuanVideo** optimizations: full offload, quantization and tiling support - **LTXVideo** optimizations: full offload, quantization and tiling support -- VAE tiling granular options in *settings -> variable auto encoder* -- UI: live preview optimizations and error handling -- UI: live preview sigma calulations, thanks @Disty0 -- UI: CSS optimizations when log view is disabled -- Samplers: add flow shift options and separate dynamic thresholding from dynamic shifting -- Refactor: remove all LDM imports if running in native mode +- **VAE**: tiling granular options in *settings -> variable auto encoder* +- **UI**: + - live preview optimizations and error handling + - live preview high quality for flow models, thanks @Disty0 + - CSS optimizations when log view is disabled +- **Samplers**: + - add flow shift options and separate dynamic thresholding from dynamic shifting + - autodetect matching sigma capabilities +- **Refactor**: + - remove all LDM imports if running in native mode - **Fixes** - do not show disabled networks - image width/height calculation when doing img2img diff --git a/modules/sd_samplers.py b/modules/sd_samplers.py index 252e52d0f..89c4e5edf 100644 --- a/modules/sd_samplers.py +++ b/modules/sd_samplers.py @@ -92,9 +92,12 @@ def create_sampler(name, model): sampler = config.constructor(model) if sampler is None: sampler = config.constructor(model) + if sampler is None or sampler.sampler is None: + model.scheduler = copy.deepcopy(model.default_scheduler) + else: + model.scheduler = sampler.sampler if not hasattr(model, 'scheduler_config'): - model.scheduler_config = sampler.sampler.config.copy() if hasattr(sampler.sampler, 'config') else {} - model.scheduler = sampler.sampler + model.scheduler_config = sampler.sampler.config.copy() if hasattr(sampler, 'sampler') and hasattr(sampler.sampler, 'config') else {} if hasattr(model, "prior_pipe") and hasattr(model.prior_pipe, "scheduler"): model.prior_pipe.scheduler = sampler.sampler model.prior_pipe.scheduler.config.clip_sample = False @@ -102,8 +105,9 @@ def create_sampler(name, model): shared.state.prediction_type = "flow_prediction" elif hasattr(model.scheduler, "config") and hasattr(model.scheduler.config, "prediction_type"): shared.state.prediction_type = model.scheduler.config.prediction_type - clean_config = {k: v for k, v in sampler.config.items() if v is not None and v is not False} - shared.log.debug(f'Sampler: "{sampler.name}" class={model.scheduler.__class__.__name__} config={clean_config}') + clean_config = {k: v for k, v in model.scheduler.config.items() if not k.startswith('_') and v is not None and v is not False} + name = sampler.name if sampler is not None and sampler.sampler is not None else 'Default' + shared.log.debug(f'Sampler: "{name}" class={model.scheduler.__class__.__name__} config={clean_config}') return sampler.sampler else: return None diff --git a/modules/sd_samplers_diffusers.py b/modules/sd_samplers_diffusers.py index 13a5c3ad9..06459d8a8 100644 --- a/modules/sd_samplers_diffusers.py +++ b/modules/sd_samplers_diffusers.py @@ -268,7 +268,15 @@ def __init__(self, name, constructor, model, **kwargs): debug(f'Sampler: config={self.config}') debug(f'Sampler: signature={possible}') # shared.log.debug(f'Sampler: sampler="{name}" config={self.config}') - self.sampler = constructor(**self.config) + sampler = constructor(**self.config) + accept_sigmas = "sigmas" in set(inspect.signature(sampler.set_timesteps).parameters.keys()) + accepts_timesteps = "timesteps" in set(inspect.signature(sampler.set_timesteps).parameters.keys()) + debug(f'Sampler: sampler="{name}" sigmas={accept_sigmas} timesteps={accepts_timesteps}') + if ('Flux' in model.__class__.__name__) and (not accept_sigmas): + shared.log.warning(f'Sampler: sampler="{name}" does not accept sigmas') + self.sampler = None + return + self.sampler = sampler if name == 'DC Solver': if not hasattr(self.sampler, 'dc_ratios'): pass From 428a5f2f1fb435ed43541bc1648789ab138e3682 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 30 Dec 2024 12:40:35 -0500 Subject: [PATCH 184/249] fix flux unet override load with bnb Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 7 +++++-- modules/model_flux.py | 18 ++++++++++++++---- modules/processing_class.py | 4 ++-- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 06524e1c8..bb4d73c46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,11 +12,13 @@ - **VAE**: tiling granular options in *settings -> variable auto encoder* - **UI**: - live preview optimizations and error handling - - live preview high quality for flow models, thanks @Disty0 + - live preview high quality output, thanks @Disty0 - CSS optimizations when log view is disabled - **Samplers**: - add flow shift options and separate dynamic thresholding from dynamic shifting - autodetect matching sigma capabilities +- **API** + - better default values for generate - **Refactor**: - remove all LDM imports if running in native mode - **Fixes** @@ -29,7 +31,8 @@ - xyz grid recover on error - hires batch - sdxl refiner - - kandinsky + - kandinsky + - flux custom unet loader for bnb ## Update for 2024-12-24 diff --git a/modules/model_flux.py b/modules/model_flux.py index 7a3fa9e00..e3adf3981 100644 --- a/modules/model_flux.py +++ b/modules/model_flux.py @@ -212,7 +212,8 @@ def load_transformer(file_path): # triggered by opts.sd_unet change _transformer, _text_encoder_2 = load_flux_quanto(file_path) if _transformer is not None: transformer = _transformer - elif quant == 'fp8' or quant == 'fp4' or quant == 'nf4': + elif quant == 'fp8' or quant == 'fp4' or quant == 'nf4' or 'Model' in shared.opts.bnb_quantization: + print('HERE0') _transformer, _text_encoder_2 = load_flux_bnb(file_path, diffusers_load_config) if _transformer is not None: transformer = _transformer @@ -222,9 +223,18 @@ def load_transformer(file_path): # triggered by opts.sd_unet change if _transformer is not None: transformer = _transformer else: - diffusers_load_config = model_quant.create_bnb_config(diffusers_load_config) - diffusers_load_config = model_quant.create_ao_config(diffusers_load_config) - transformer = diffusers.FluxTransformer2DModel.from_single_file(file_path, **diffusers_load_config) + print('HERE1') + quant_args = {} + quant_args = model_quant.create_bnb_config(quant_args) + if quant_args: + model_quant.load_bnb(f'Load model: type=Sana quant={quant_args}') + if not quant_args: + quant_args = model_quant.create_ao_config(quant_args) + if quant_args: + model_quant.load_torchao(f'Load model: type=Sana quant={quant_args}') + print('HERE2', diffusers_load_config) + print('HERE3', quant_args) + transformer = diffusers.FluxTransformer2DModel.from_single_file(file_path, **diffusers_load_config, **quant_args) if transformer is None: shared.log.error('Failed to load UNet model') shared.opts.sd_unet = 'None' diff --git a/modules/processing_class.py b/modules/processing_class.py index 97e5f7b6f..1f8382128 100644 --- a/modules/processing_class.py +++ b/modules/processing_class.py @@ -29,7 +29,7 @@ def __init__(self, seed_resize_from_w: int = -1, batch_size: int = 1, n_iter: int = 1, - steps: int = 50, + steps: int = 20, clip_skip: int = 1, width: int = 1024, height: int = 1024, @@ -39,7 +39,7 @@ def __init__(self, hr_sampler_name: str = None, eta: float = None, # guidance - cfg_scale: float = 7.0, + cfg_scale: float = 6.0, cfg_end: float = 1, diffusers_guidance_rescale: float = 0.7, pag_scale: float = 0.0, From 62789e2ba93cdf0d1a112cb47a7d6e7dd6b5301a Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 30 Dec 2024 12:43:42 -0500 Subject: [PATCH 185/249] cleanup Signed-off-by: Vladimir Mandic --- modules/model_flux.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/modules/model_flux.py b/modules/model_flux.py index e3adf3981..b89714d91 100644 --- a/modules/model_flux.py +++ b/modules/model_flux.py @@ -213,7 +213,6 @@ def load_transformer(file_path): # triggered by opts.sd_unet change if _transformer is not None: transformer = _transformer elif quant == 'fp8' or quant == 'fp4' or quant == 'nf4' or 'Model' in shared.opts.bnb_quantization: - print('HERE0') _transformer, _text_encoder_2 = load_flux_bnb(file_path, diffusers_load_config) if _transformer is not None: transformer = _transformer @@ -223,7 +222,6 @@ def load_transformer(file_path): # triggered by opts.sd_unet change if _transformer is not None: transformer = _transformer else: - print('HERE1') quant_args = {} quant_args = model_quant.create_bnb_config(quant_args) if quant_args: @@ -232,8 +230,6 @@ def load_transformer(file_path): # triggered by opts.sd_unet change quant_args = model_quant.create_ao_config(quant_args) if quant_args: model_quant.load_torchao(f'Load model: type=Sana quant={quant_args}') - print('HERE2', diffusers_load_config) - print('HERE3', quant_args) transformer = diffusers.FluxTransformer2DModel.from_single_file(file_path, **diffusers_load_config, **quant_args) if transformer is None: shared.log.error('Failed to load UNet model') From 864803398299c4837f18365c934e16eb12e1df59 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 30 Dec 2024 14:08:14 -0500 Subject: [PATCH 186/249] teacache for ltxvideo Signed-off-by: Vladimir Mandic --- modules/sd_samplers_common.py | 9 +- modules/teacache/teacache_ltx.py | 163 +++++++++++++++++++++++++++++++ scripts/ltxvideo.py | 20 +++- 3 files changed, 186 insertions(+), 6 deletions(-) create mode 100644 modules/teacache/teacache_ltx.py diff --git a/modules/sd_samplers_common.py b/modules/sd_samplers_common.py index c4a2712f3..b01b847d8 100644 --- a/modules/sd_samplers_common.py +++ b/modules/sd_samplers_common.py @@ -52,9 +52,12 @@ def single_sample_to_image(sample, approximation=None): if len(sample.shape) == 4 and sample.shape[0]: # likely animatediff latent sample = sample.permute(1, 0, 2, 3)[0] if approximation == 2: # TAESD - if shared.opts.live_preview_downscale and (sample.shape[-1] > 128 or sample.shape[-2] > 128): - scale = 128 / max(sample.shape[-1], sample.shape[-2]) - sample = torch.nn.functional.interpolate(sample.unsqueeze(0), scale_factor=[scale, scale], mode='bilinear', align_corners=False)[0] + if (len(sample.shape) == 3 or len(sample.shape) == 4) and shared.opts.live_preview_downscale and (sample.shape[-1] > 128 or sample.shape[-2] > 128): + try: + scale = 128 / max(sample.shape[-1], sample.shape[-2]) + sample = torch.nn.functional.interpolate(sample.unsqueeze(0), scale_factor=[scale, scale], mode='bilinear', align_corners=False)[0] + except Exception: + pass x_sample = sd_vae_taesd.decode(sample) x_sample = (1.0 + x_sample) / 2.0 # preview requires smaller range elif shared.sd_model_type == 'sc' and approximation != 3: diff --git a/modules/teacache/teacache_ltx.py b/modules/teacache/teacache_ltx.py new file mode 100644 index 000000000..d0e3c942e --- /dev/null +++ b/modules/teacache/teacache_ltx.py @@ -0,0 +1,163 @@ +from typing import Any, Dict, Optional, Tuple +import numpy as np +import torch +from diffusers.models.modeling_outputs import Transformer2DModelOutput +from diffusers.utils import is_torch_version, scale_lora_layers, unscale_lora_layers + + +def teacache_forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor, + timestep: torch.LongTensor, + encoder_attention_mask: torch.Tensor, + num_frames: int, + height: int, + width: int, + rope_interpolation_scale: Optional[Tuple[float, float, float]] = None, + attention_kwargs: Optional[Dict[str, Any]] = None, + return_dict: bool = True, + ) -> torch.Tensor: + if attention_kwargs is not None: + attention_kwargs = attention_kwargs.copy() + lora_scale = attention_kwargs.pop("scale", 1.0) + else: + lora_scale = 1.0 + + scale_lora_layers(self, lora_scale) + + image_rotary_emb = self.rope(hidden_states, num_frames, height, width, rope_interpolation_scale) + + # convert encoder_attention_mask to a bias the same way we do for attention_mask + if encoder_attention_mask is not None and encoder_attention_mask.ndim == 2: + encoder_attention_mask = (1 - encoder_attention_mask.to(hidden_states.dtype)) * -10000.0 + encoder_attention_mask = encoder_attention_mask.unsqueeze(1) + + batch_size = hidden_states.size(0) + hidden_states = self.proj_in(hidden_states) + + temb, embedded_timestep = self.time_embed( + timestep.flatten(), + batch_size=batch_size, + hidden_dtype=hidden_states.dtype, + ) + + temb = temb.view(batch_size, -1, temb.size(-1)) + embedded_timestep = embedded_timestep.view(batch_size, -1, embedded_timestep.size(-1)) + + encoder_hidden_states = self.caption_projection(encoder_hidden_states) + encoder_hidden_states = encoder_hidden_states.view(batch_size, -1, hidden_states.size(-1)) + + if self.enable_teacache: + inp = hidden_states.clone() + temb_ = temb.clone() + inp = self.transformer_blocks[0].norm1(inp) + num_ada_params = self.transformer_blocks[0].scale_shift_table.shape[0] + ada_values = self.transformer_blocks[0].scale_shift_table[None, None] + temb_.reshape(batch_size, temb_.size(1), num_ada_params, -1) + shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = ada_values.unbind(dim=2) + modulated_inp = inp * (1 + scale_msa) + shift_msa + if self.cnt == 0 or self.cnt == self.num_steps-1: + should_calc = True + self.accumulated_rel_l1_distance = 0 + else: + coefficients = [2.14700694e+01, -1.28016453e+01, 2.31279151e+00, 7.92487521e-01, 9.69274326e-03] + rescale_func = np.poly1d(coefficients) + self.accumulated_rel_l1_distance += rescale_func(((modulated_inp-self.previous_modulated_input).abs().mean() / self.previous_modulated_input.abs().mean()).cpu().item()) + if self.accumulated_rel_l1_distance < self.rel_l1_thresh: + should_calc = False + else: + should_calc = True + self.accumulated_rel_l1_distance = 0 + self.previous_modulated_input = modulated_inp + self.cnt += 1 + if self.cnt == self.num_steps: + self.cnt = 0 + + if self.enable_teacache: + if not should_calc: + hidden_states += self.previous_residual + else: + ori_hidden_states = hidden_states.clone() + for block in self.transformer_blocks: + if torch.is_grad_enabled() and self.gradient_checkpointing: + + def create_custom_forward(module, return_dict=None): + def custom_forward(*inputs): + if return_dict is not None: + return module(*inputs, return_dict=return_dict) + else: + return module(*inputs) + + return custom_forward + + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(block), + hidden_states, + encoder_hidden_states, + temb, + image_rotary_emb, + encoder_attention_mask, + **ckpt_kwargs, + ) + else: + hidden_states = block( + hidden_states=hidden_states, + encoder_hidden_states=encoder_hidden_states, + temb=temb, + image_rotary_emb=image_rotary_emb, + encoder_attention_mask=encoder_attention_mask, + ) + + scale_shift_values = self.scale_shift_table[None, None] + embedded_timestep[:, :, None] + shift, scale = scale_shift_values[:, :, 0], scale_shift_values[:, :, 1] + + hidden_states = self.norm_out(hidden_states) + hidden_states = hidden_states * (1 + scale) + shift + self.previous_residual = hidden_states - ori_hidden_states + else: + for block in self.transformer_blocks: + if torch.is_grad_enabled() and self.gradient_checkpointing: + + def create_custom_forward(module, return_dict=None): + def custom_forward(*inputs): + if return_dict is not None: + return module(*inputs, return_dict=return_dict) + else: + return module(*inputs) + + return custom_forward + + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(block), + hidden_states, + encoder_hidden_states, + temb, + image_rotary_emb, + encoder_attention_mask, + **ckpt_kwargs, + ) + else: + hidden_states = block( + hidden_states=hidden_states, + encoder_hidden_states=encoder_hidden_states, + temb=temb, + image_rotary_emb=image_rotary_emb, + encoder_attention_mask=encoder_attention_mask, + ) + + scale_shift_values = self.scale_shift_table[None, None] + embedded_timestep[:, :, None] + shift, scale = scale_shift_values[:, :, 0], scale_shift_values[:, :, 1] + + hidden_states = self.norm_out(hidden_states) + hidden_states = hidden_states * (1 + scale) + shift + + + output = self.proj_out(hidden_states) + + unscale_lora_layers(self, lora_scale) + + if not return_dict: + return (output,) + return Transformer2DModelOutput(sample=output) diff --git a/scripts/ltxvideo.py b/scripts/ltxvideo.py index 7b12a3a86..e6464970d 100644 --- a/scripts/ltxvideo.py +++ b/scripts/ltxvideo.py @@ -5,6 +5,7 @@ import diffusers import transformers from modules import scripts, processing, shared, images, devices, sd_models, sd_checkpoint, model_quant, timer +from modules.teacache.teacache_ltx import teacache_forward repos = { @@ -83,6 +84,9 @@ def model_change(model): with gr.Row(): num_frames = gr.Slider(label='Frames', minimum=9, maximum=257, step=1, value=41) sampler = gr.Checkbox(label='Override sampler', value=True) + with gr.Row(): + teacache_enable = gr.Checkbox(label='Enable TeaCache', value=False) + teacache_threshold = gr.Slider(label='Threshold', minimum=0.01, maximum=0.1, step=0.01, value=0.03) with gr.Row(): model_custom = gr.Textbox(value='', label='Path to model file', visible=False) with gr.Row(): @@ -94,9 +98,9 @@ def model_change(model): mp4_interpolate = gr.Slider(label='Interpolate frames', minimum=0, maximum=24, step=1, value=0, visible=False) video_type.change(fn=video_type_change, inputs=[video_type], outputs=[duration, gif_loop, mp4_pad, mp4_interpolate]) model.change(fn=model_change, inputs=[model], outputs=[model_custom]) - return [model, model_custom, decode, sampler, num_frames, video_type, duration, gif_loop, mp4_pad, mp4_interpolate] + return [model, model_custom, decode, sampler, num_frames, video_type, duration, gif_loop, mp4_pad, mp4_interpolate, teacache_enable, teacache_threshold] - def run(self, p: processing.StableDiffusionProcessing, model, model_custom, decode, sampler, num_frames, video_type, duration, gif_loop, mp4_pad, mp4_interpolate): # pylint: disable=arguments-differ, unused-argument + def run(self, p: processing.StableDiffusionProcessing, model, model_custom, decode, sampler, num_frames, video_type, duration, gif_loop, mp4_pad, mp4_interpolate, teacache_enable, teacache_threshold): # pylint: disable=arguments-differ, unused-argument # set params image = getattr(p, 'init_images', None) image = None if image is None or len(image) == 0 else image[0] @@ -130,6 +134,7 @@ def run(self, p: processing.StableDiffusionProcessing, model, model_custom, deco kwargs = {} kwargs = model_quant.create_bnb_config(kwargs) kwargs = model_quant.create_ao_config(kwargs) + diffusers.LTXVideoTransformer3DModel.forward = teacache_forward if os.path.isfile(repo_id): shared.sd_model = cls.from_single_file( repo_id, @@ -156,7 +161,16 @@ def run(self, p: processing.StableDiffusionProcessing, model, model_custom, deco shared.sd_model.vae.enable_slicing() shared.sd_model.vae.enable_tiling() devices.torch_gc(force=True) - shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} args={p.task_args}') + + shared.sd_model.transformer.cnt = 0 + shared.sd_model.transformer.accumulated_rel_l1_distance = 0 + shared.sd_model.transformer.previous_modulated_input = None + shared.sd_model.transformer.previous_residual = None + shared.sd_model.transformer.enable_teacache = teacache_enable + shared.sd_model.transformer.rel_l1_thresh = teacache_threshold + shared.sd_model.transformer.num_steps = p.steps + + shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} args={p.task_args} steps={p.steps} teacache={teacache_enable} threshold={teacache_threshold}') # run processing t0 = time.time() From 9263b1b565f33c52402f621b73e3c8eb922f071e Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 30 Dec 2024 14:19:15 -0500 Subject: [PATCH 187/249] update changelog Signed-off-by: Vladimir Mandic --- .pylintrc | 1 + .ruff.toml | 1 + CHANGELOG.md | 10 +++++++--- modules/teacache/teacache_ltx.py | 4 ++++ 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.pylintrc b/.pylintrc index ad42ddd13..4a2850664 100644 --- a/.pylintrc +++ b/.pylintrc @@ -31,6 +31,7 @@ ignore-paths=/usr/lib/.*$, modules/rife, modules/schedulers, modules/taesd, + modules/teacache, modules/todo, modules/unipc, modules/xadapter, diff --git a/.ruff.toml b/.ruff.toml index 4bab64260..a2ad0b91a 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -26,6 +26,7 @@ exclude = [ "modules/schedulers", "modules/segmoe", "modules/taesd", + "modules/teacache", "modules/todo", "modules/unipc", "modules/xadapter", diff --git a/CHANGELOG.md b/CHANGELOG.md index bb4d73c46..06c072fbd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,9 +7,13 @@ - quantized models support - fuse support with on-demand apply/unapply - add legacy option in *settings -> networks* -- **HunyuanVideo** optimizations: full offload, quantization and tiling support -- **LTXVideo** optimizations: full offload, quantization and tiling support -- **VAE**: tiling granular options in *settings -> variable auto encoder* +- **HunyuanVideo** + - optimizations: full offload, quantization and tiling support +- **LTXVideo** + - optimizations: full offload, quantization and tiling support + - [TeaCache](https://github.com/ali-vilab/TeaCache/blob/main/TeaCache4LTX-Video/README.md) integration +- **VAE**: + - tiling granular options in *settings -> variable auto encoder* - **UI**: - live preview optimizations and error handling - live preview high quality output, thanks @Disty0 diff --git a/modules/teacache/teacache_ltx.py b/modules/teacache/teacache_ltx.py index d0e3c942e..f7f9cd83d 100644 --- a/modules/teacache/teacache_ltx.py +++ b/modules/teacache/teacache_ltx.py @@ -1,3 +1,7 @@ +""" +source: https://github.com/ali-vilab/TeaCache/blob/main/TeaCache4LTX-Video/teacache_ltx.py +""" + from typing import Any, Dict, Optional, Tuple import numpy as np import torch From 741566d885246d67a2fac161859c9af87f60747a Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 30 Dec 2024 14:41:53 -0500 Subject: [PATCH 188/249] cleanup hdr logging Signed-off-by: Vladimir Mandic --- modules/processing_args.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/modules/processing_args.py b/modules/processing_args.py index 244948530..34e93e8d4 100644 --- a/modules/processing_args.py +++ b/modules/processing_args.py @@ -343,13 +343,7 @@ def set_pipeline_args(p, model, prompts:list, negative_prompts:list, prompts_2:t shared.log.info(f'{desc}: pipeline={model.__class__.__name__} task={sd_models.get_diffusers_task(model)} batch={p.iteration + 1}/{p.n_iter}x{p.batch_size} set={clean}') if p.hdr_clamp or p.hdr_maximize or p.hdr_brightness != 0 or p.hdr_color != 0 or p.hdr_sharpen != 0: - txt = 'HDR:' - txt += f' Brightness={p.hdr_brightness}' if p.hdr_brightness != 0 else ' Brightness off' - txt += f' Color={p.hdr_color}' if p.hdr_color != 0 else ' Color off' - txt += f' Sharpen={p.hdr_sharpen}' if p.hdr_sharpen != 0 else ' Sharpen off' - txt += f' Clamp threshold={p.hdr_threshold} boundary={p.hdr_boundary}' if p.hdr_clamp else ' Clamp off' - txt += f' Maximize boundary={p.hdr_max_boundry} center={p.hdr_max_center}' if p.hdr_maximize else ' Maximize off' - shared.log.debug(txt) + shared.log.debug(f'HDR: clamp={p.hdr_clamp} maximize={p.hdr_maximize} brightness={p.hdr_brightness} color={p.hdr_color} sharpen={p.hdr_sharpen} threshold={p.hdr_threshold} boundary={p.hdr_boundary} max={p.hdr_max_boundry} center={p.hdr_max_center}') if shared.cmd_opts.profile: t1 = time.time() shared.log.debug(f'Profile: pipeline args: {t1-t0:.2f}') From 3474dd15c15f142367aed7a139c6ef904e637bb6 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 30 Dec 2024 14:46:23 -0500 Subject: [PATCH 189/249] update changelog Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 06c072fbd..f85e6dbbd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,18 +26,18 @@ - **Refactor**: - remove all LDM imports if running in native mode - **Fixes** - - do not show disabled networks - - image width/height calculation when doing img2img - flux pipeline switches: txt/img/inpaint + - flux custom unet loader for bnb - interrogate caption with T5 - on-the-fly quantization using TorchAO - remove concurrent preview requests - xyz grid recover on error - hires batch - sdxl refiner - - kandinsky - - flux custom unet loader for bnb - + - kandinsky matmul + - do not show disabled networks + - image width/height calculation when doing img2img + ## Update for 2024-12-24 ### Highlights for 2024-12-24 From 53395318c723b2dbcd51db909f05fb3b7d92f0cf Mon Sep 17 00:00:00 2001 From: Disty0 Date: Tue, 31 Dec 2024 04:15:37 +0300 Subject: [PATCH 190/249] OpenVINO fix Lora --- modules/lora/networks.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/modules/lora/networks.py b/modules/lora/networks.py index 0be0a25f7..ebc30b6dd 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -146,6 +146,7 @@ def load_safetensors(name, network_on_disk) -> Union[network.Network, None]: def maybe_recompile_model(names, te_multipliers): recompile_model = False + skip_lora_load = False if shared.compiled_model_state is not None and shared.compiled_model_state.is_compiled: if len(names) == len(shared.compiled_model_state.lora_model): for i, name in enumerate(names): @@ -155,19 +156,23 @@ def maybe_recompile_model(names, te_multipliers): shared.compiled_model_state.lora_model = [] break if not recompile_model: + skip_lora_load = True if len(loaded_networks) > 0 and debug: shared.log.debug('Model Compile: Skipping LoRa loading') - return recompile_model + return recompile_model, skip_lora_load else: recompile_model = True shared.compiled_model_state.lora_model = [] if recompile_model: backup_cuda_compile = shared.opts.cuda_compile + backup_scheduler = getattr(shared.sd_model, "scheduler", None) sd_models.unload_model_weights(op='model') shared.opts.cuda_compile = [] sd_models.reload_model_weights(op='model') shared.opts.cuda_compile = backup_cuda_compile - return recompile_model + if backup_scheduler is not None: + shared.sd_model.scheduler = backup_scheduler + return recompile_model, skip_lora_load def list_available_networks(): @@ -230,7 +235,7 @@ def network_load(names, te_multipliers=None, unet_multipliers=None, dyn_dims=Non if names[i].startswith('/'): networks_on_disk[i] = network_download(names[i]) failed_to_load_networks = [] - recompile_model = maybe_recompile_model(names, te_multipliers) + recompile_model, skip_lora_load = maybe_recompile_model(names, te_multipliers) loaded_networks.clear() diffuser_loaded.clear() @@ -272,7 +277,7 @@ def network_load(names, te_multipliers=None, unet_multipliers=None, dyn_dims=Non name = next(iter(lora_cache)) lora_cache.pop(name, None) - if len(diffuser_loaded) > 0: + if not skip_lora_load and len(diffuser_loaded) > 0: shared.log.debug(f'Load network: type=LoRA loaded={diffuser_loaded} available={shared.sd_model.get_list_adapters()} active={shared.sd_model.get_active_adapters()} scales={diffuser_scales}') try: t0 = time.time() @@ -294,7 +299,6 @@ def network_load(names, te_multipliers=None, unet_multipliers=None, dyn_dims=Non backup_lora_model = shared.compiled_model_state.lora_model if 'Model' in shared.opts.cuda_compile: shared.sd_model = sd_models_compile.compile_diffusers(shared.sd_model) - shared.compiled_model_state.lora_model = backup_lora_model if len(loaded_networks) > 0: @@ -498,7 +502,7 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn def network_deactivate(include=[], exclude=[]): - if not shared.opts.lora_fuse_diffusers: + if not shared.opts.lora_fuse_diffusers or shared.opts.lora_force_diffusers: return t0 = time.time() sd_model = getattr(shared.sd_model, "pipe", shared.sd_model) # wrapped model compatiblility From c584ca9c00ea54fc0cfd7ed85503771e5db852e0 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 30 Dec 2024 19:12:19 -0500 Subject: [PATCH 191/249] increase progress timeout Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 1 + javascript/progressBar.js | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f85e6dbbd..8fcd3c707 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ - xyz grid recover on error - hires batch - sdxl refiner + - increase progress timeout - kandinsky matmul - do not show disabled networks - image width/height calculation when doing img2img diff --git a/javascript/progressBar.js b/javascript/progressBar.js index 3f5c9fcef..27404b440 100644 --- a/javascript/progressBar.js +++ b/javascript/progressBar.js @@ -141,7 +141,7 @@ function requestProgress(id_task, progressEl, galleryEl, atEnd = null, onProgres done(); }; - xhrPost('./internal/progress', { id_task, id_live_preview }, onProgressHandler, onProgressErrorHandler, false, 5000); + xhrPost('./internal/progress', { id_task, id_live_preview }, onProgressHandler, onProgressErrorHandler, false, 30000); }; debug('livePreview start:', dateStart); start(id_task, 0); From 6c49b50805165f5abe0098e8ebfa56f65d869535 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 30 Dec 2024 19:54:28 -0500 Subject: [PATCH 192/249] enable debug by default and optimize startup Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 2 ++ installer.py | 55 +++++++++++++++++++++---------------------- modules/cmd_args.py | 28 ++++++++++++---------- modules/extensions.py | 2 +- modules/shared.py | 1 - modules/theme.py | 7 ++---- scripts/k_diff.py | 2 +- webui.py | 10 +++++--- 8 files changed, 55 insertions(+), 52 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fcd3c707..832ae2dc1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ - better default values for generate - **Refactor**: - remove all LDM imports if running in native mode + - startup optimizatios - **Fixes** - flux pipeline switches: txt/img/inpaint - flux custom unet loader for bnb @@ -37,6 +38,7 @@ - increase progress timeout - kandinsky matmul - do not show disabled networks + - enable debug logging by default - image width/height calculation when doing img2img ## Update for 2024-12-24 diff --git a/installer.py b/installer.py index 43c4e84e3..eb6eb8511 100644 --- a/installer.py +++ b/installer.py @@ -1307,44 +1307,43 @@ def check_timestamp(): def add_args(parser): group_setup = parser.add_argument_group('Setup') - group_setup.add_argument('--reset', default = os.environ.get("SD_RESET",False), action='store_true', help = "Reset main repository to latest version, default: %(default)s") - group_setup.add_argument('--upgrade', '--update', default = os.environ.get("SD_UPGRADE",False), action='store_true', help = "Upgrade main repository to latest version, default: %(default)s") - group_setup.add_argument('--requirements', default = os.environ.get("SD_REQUIREMENTS",False), action='store_true', help = "Force re-check of requirements, default: %(default)s") - group_setup.add_argument('--reinstall', default = os.environ.get("SD_REINSTALL",False), action='store_true', help = "Force reinstallation of all requirements, default: %(default)s") - group_setup.add_argument('--optional', default = os.environ.get("SD_OPTIONAL",False), action='store_true', help = "Force installation of optional requirements, default: %(default)s") - group_setup.add_argument('--uv', default = os.environ.get("SD_UV",False), action='store_true', help = "Use uv instead of pip to install the packages") + group_setup.add_argument('--reset', default=os.environ.get("SD_RESET",False), action='store_true', help="Reset main repository to latest version, default: %(default)s") + group_setup.add_argument('--upgrade', '--update', default=os.environ.get("SD_UPGRADE",False), action='store_true', help="Upgrade main repository to latest version, default: %(default)s") + group_setup.add_argument('--requirements', default=os.environ.get("SD_REQUIREMENTS",False), action='store_true', help="Force re-check of requirements, default: %(default)s") + group_setup.add_argument('--reinstall', default=os.environ.get("SD_REINSTALL",False), action='store_true', help="Force reinstallation of all requirements, default: %(default)s") + group_setup.add_argument('--optional', default=os.environ.get("SD_OPTIONAL",False), action='store_true', help="Force installation of optional requirements, default: %(default)s") + group_setup.add_argument('--uv', default=os.environ.get("SD_UV",False), action='store_true', help="Use uv instead of pip to install the packages") group_startup = parser.add_argument_group('Startup') - group_startup.add_argument('--quick', default = os.environ.get("SD_QUICK",False), action='store_true', help = "Bypass version checks, default: %(default)s") - group_startup.add_argument('--skip-requirements', default = os.environ.get("SD_SKIPREQUIREMENTS",False), action='store_true', help = "Skips checking and installing requirements, default: %(default)s") - group_startup.add_argument('--skip-extensions', default = os.environ.get("SD_SKIPEXTENSION",False), action='store_true', help = "Skips running individual extension installers, default: %(default)s") - group_startup.add_argument('--skip-git', default = os.environ.get("SD_SKIPGIT",False), action='store_true', help = "Skips running all GIT operations, default: %(default)s") - group_startup.add_argument('--skip-torch', default = os.environ.get("SD_SKIPTORCH",False), action='store_true', help = "Skips running Torch checks, default: %(default)s") - group_startup.add_argument('--skip-all', default = os.environ.get("SD_SKIPALL",False), action='store_true', help = "Skips running all checks, default: %(default)s") - group_startup.add_argument('--skip-env', default = os.environ.get("SD_SKIPENV",False), action='store_true', help = "Skips setting of env variables during startup, default: %(default)s") + group_startup.add_argument('--quick', default=os.environ.get("SD_QUICK",False), action='store_true', help="Bypass version checks, default: %(default)s") + group_startup.add_argument('--skip-requirements', default=os.environ.get("SD_SKIPREQUIREMENTS",False), action='store_true', help="Skips checking and installing requirements, default: %(default)s") + group_startup.add_argument('--skip-extensions', default=os.environ.get("SD_SKIPEXTENSION",False), action='store_true', help="Skips running individual extension installers, default: %(default)s") + group_startup.add_argument('--skip-git', default=os.environ.get("SD_SKIPGIT",False), action='store_true', help="Skips running all GIT operations, default: %(default)s") + group_startup.add_argument('--skip-torch', default=os.environ.get("SD_SKIPTORCH",False), action='store_true', help="Skips running Torch checks, default: %(default)s") + group_startup.add_argument('--skip-all', default=os.environ.get("SD_SKIPALL",False), action='store_true', help="Skips running all checks, default: %(default)s") + group_startup.add_argument('--skip-env', default=os.environ.get("SD_SKIPENV",False), action='store_true', help="Skips setting of env variables during startup, default: %(default)s") group_compute = parser.add_argument_group('Compute Engine') - group_compute.add_argument('--use-directml', default = os.environ.get("SD_USEDIRECTML",False), action='store_true', help = "Use DirectML if no compatible GPU is detected, default: %(default)s") - group_compute.add_argument("--use-openvino", default = os.environ.get("SD_USEOPENVINO",False), action='store_true', help="Use Intel OpenVINO backend, default: %(default)s") - group_compute.add_argument("--use-ipex", default = os.environ.get("SD_USEIPEX",False), action='store_true', help="Force use Intel OneAPI XPU backend, default: %(default)s") - group_compute.add_argument("--use-cuda", default = os.environ.get("SD_USECUDA",False), action='store_true', help="Force use nVidia CUDA backend, default: %(default)s") - group_compute.add_argument("--use-rocm", default = os.environ.get("SD_USEROCM",False), action='store_true', help="Force use AMD ROCm backend, default: %(default)s") - group_compute.add_argument('--use-zluda', default=os.environ.get("SD_USEZLUDA", False), action='store_true', help = "Force use ZLUDA, AMD GPUs only, default: %(default)s") - group_compute.add_argument("--use-xformers", default = os.environ.get("SD_USEXFORMERS",False), action='store_true', help="Force use xFormers cross-optimization, default: %(default)s") + group_compute.add_argument('--use-directml', default=os.environ.get("SD_USEDIRECTML",False), action='store_true', help="Use DirectML if no compatible GPU is detected, default: %(default)s") + group_compute.add_argument("--use-openvino", default=os.environ.get("SD_USEOPENVINO",False), action='store_true', help="Use Intel OpenVINO backend, default: %(default)s") + group_compute.add_argument("--use-ipex", default=os.environ.get("SD_USEIPEX",False), action='store_true', help="Force use Intel OneAPI XPU backend, default: %(default)s") + group_compute.add_argument("--use-cuda", default=os.environ.get("SD_USECUDA",False), action='store_true', help="Force use nVidia CUDA backend, default: %(default)s") + group_compute.add_argument("--use-rocm", default=os.environ.get("SD_USEROCM",False), action='store_true', help="Force use AMD ROCm backend, default: %(default)s") + group_compute.add_argument('--use-zluda', default=os.environ.get("SD_USEZLUDA", False), action='store_true', help="Force use ZLUDA, AMD GPUs only, default: %(default)s") + group_compute.add_argument("--use-xformers", default=os.environ.get("SD_USEXFORMERS",False), action='store_true', help="Force use xFormers cross-optimization, default: %(default)s") group_diag = parser.add_argument_group('Diagnostics') - group_diag.add_argument('--safe', default = os.environ.get("SD_SAFE",False), action='store_true', help = "Run in safe mode with no user extensions") - group_diag.add_argument('--experimental', default = os.environ.get("SD_EXPERIMENTAL",False), action='store_true', help = "Allow unsupported versions of libraries, default: %(default)s") - group_diag.add_argument('--test', default = os.environ.get("SD_TEST",False), action='store_true', help = "Run test only and exit") - group_diag.add_argument('--version', default = False, action='store_true', help = "Print version information") - group_diag.add_argument('--ignore', default = os.environ.get("SD_IGNORE",False), action='store_true', help = "Ignore any errors and attempt to continue") + group_diag.add_argument('--safe', default=os.environ.get("SD_SAFE",False), action='store_true', help="Run in safe mode with no user extensions") + group_diag.add_argument('--experimental', default=os.environ.get("SD_EXPERIMENTAL",False), action='store_true', help="Allow unsupported versions of libraries, default: %(default)s") + group_diag.add_argument('--test', default=os.environ.get("SD_TEST",False), action='store_true', help="Run test only and exit") + group_diag.add_argument('--version', default=False, action='store_true', help="Print version information") + group_diag.add_argument('--ignore', default=os.environ.get("SD_IGNORE",False), action='store_true', help="Ignore any errors and attempt to continue") group_log = parser.add_argument_group('Logging') group_log.add_argument("--log", type=str, default=os.environ.get("SD_LOG", None), help="Set log file, default: %(default)s") - group_log.add_argument('--debug', default = os.environ.get("SD_DEBUG",False), action='store_true', help = "Run installer with debug logging, default: %(default)s") + # group_log.add_argument('--debug', default=os.environ.get("SD_DEBUG",False), action='store_true', help="Run installer with debug logging, default: %(default)s") group_log.add_argument("--profile", default=os.environ.get("SD_PROFILE", False), action='store_true', help="Run profiler, default: %(default)s") - group_log.add_argument('--docs', default=os.environ.get("SD_DOCS", False), action='store_true', help = "Mount API docs, default: %(default)s") - group_log.add_argument("--api-log", default=os.environ.get("SD_APILOG", False), action='store_true', help="Enable logging of all API requests, default: %(default)s") + group_log.add_argument('--docs', default=os.environ.get("SD_DOCS", False), action='store_true', help="Mount API docs, default: %(default)s") def parse_args(parser): diff --git a/modules/cmd_args.py b/modules/cmd_args.py index a1e3473ba..9fb5a3fb7 100644 --- a/modules/cmd_args.py +++ b/modules/cmd_args.py @@ -17,6 +17,15 @@ def main_args(): group_config.add_argument("--lowvram", default=os.environ.get("SD_LOWVRAM", False), action='store_true', help="Split model components and keep only active part in VRAM, default: %(default)s") group_config.add_argument("--freeze", default=os.environ.get("SD_FREEZE", False), action='store_true', help="Disable editing settings") + group_compute = parser.add_argument_group('Compute Engine') + group_compute.add_argument("--device-id", type=str, default=os.environ.get("SD_DEVICEID", None), help="Select the default CUDA device to use, default: %(default)s") + group_compute.add_argument('--use-directml', default=os.environ.get("SD_USEDIRECTML", False), action='store_true', help = "Use DirectML if no compatible GPU is detected, default: %(default)s") + group_compute.add_argument('--use-zluda', default=os.environ.get("SD_USEZLUDA", False), action='store_true', help = "Force use ZLUDA, AMD GPUs only, default: %(default)s") + group_compute.add_argument("--use-openvino", default=os.environ.get("SD_USEOPENVINO", False), action='store_true', help="Use Intel OpenVINO backend, default: %(default)s") + group_compute.add_argument("--use-ipex", default=os.environ.get("SD_USEIPX", False), action='store_true', help="Force use Intel OneAPI XPU backend, default: %(default)s") + group_compute.add_argument("--use-cuda", default=os.environ.get("SD_USECUDA", False), action='store_true', help="Force use nVidia CUDA backend, default: %(default)s") + group_compute.add_argument("--use-rocm", default=os.environ.get("SD_USEROCM", False), action='store_true', help="Force use AMD ROCm backend, default: %(default)s") + group_paths = parser.add_argument_group('Paths') group_paths.add_argument("--ckpt", type=str, default=os.environ.get("SD_MODEL", None), help="Path to model checkpoint to load immediately, default: %(default)s") group_paths.add_argument("--data-dir", type=str, default=os.environ.get("SD_DATADIR", ''), help="Base path where all user data is stored, default: %(default)s") @@ -26,17 +35,6 @@ def main_args(): group_diag.add_argument("--no-hashing", default=os.environ.get("SD_NOHASHING", False), action='store_true', help="Disable hashing of checkpoints, default: %(default)s") group_diag.add_argument("--no-metadata", default=os.environ.get("SD_NOMETADATA", False), action='store_true', help="Disable reading of metadata from models, default: %(default)s") group_diag.add_argument("--profile", default=os.environ.get("SD_PROFILE", False), action='store_true', help="Run profiler, default: %(default)s") - group_diag.add_argument("--disable-queue", default=os.environ.get("SD_DISABLEQUEUE", False), action='store_true', help="Disable queues, default: %(default)s") - group_diag.add_argument('--debug', default=os.environ.get("SD_DEBUG", False), action='store_true', help = "Run installer with debug logging, default: %(default)s") - - group_compute = parser.add_argument_group('Compute Engine') - group_compute.add_argument('--use-directml', default=os.environ.get("SD_USEDIRECTML", False), action='store_true', help = "Use DirectML if no compatible GPU is detected, default: %(default)s") - group_compute.add_argument('--use-zluda', default=os.environ.get("SD_USEZLUDA", False), action='store_true', help = "Force use ZLUDA, AMD GPUs only, default: %(default)s") - group_compute.add_argument("--use-openvino", default=os.environ.get("SD_USEOPENVINO", False), action='store_true', help="Use Intel OpenVINO backend, default: %(default)s") - group_compute.add_argument("--use-ipex", default=os.environ.get("SD_USEIPX", False), action='store_true', help="Force use Intel OneAPI XPU backend, default: %(default)s") - group_compute.add_argument("--use-cuda", default=os.environ.get("SD_USECUDA", False), action='store_true', help="Force use nVidia CUDA backend, default: %(default)s") - group_compute.add_argument("--use-rocm", default=os.environ.get("SD_USEROCM", False), action='store_true', help="Force use AMD ROCm backend, default: %(default)s") - group_diag.add_argument("--device-id", type=str, default=os.environ.get("SD_DEVICEID", None), help="Select the default CUDA device to use, default: %(default)s") group_http = parser.add_argument_group('HTTP') group_http.add_argument('--theme', type=str, default=os.environ.get("SD_THEME", None), help='Override UI theme') @@ -60,8 +58,8 @@ def main_args(): def compatibility_args(): - group_compat = parser.add_argument_group('Compatibility options') # removed args are added here as hidden in fixed format for compatbility reasons + group_compat = parser.add_argument_group('Compatibility options') group_compat.add_argument("--allow-code", default=os.environ.get("SD_ALLOWCODE", False), action='store_true', help=argparse.SUPPRESS) group_compat.add_argument("--use-cpu", nargs='+', default=[], type=str.lower, help=argparse.SUPPRESS) group_compat.add_argument("-f", action='store_true', help=argparse.SUPPRESS) # allows running as root; implemented outside of webui @@ -76,11 +74,15 @@ def compatibility_args(): group_compat.add_argument("--disable-extension-access", default=False, action='store_true', help=argparse.SUPPRESS) group_compat.add_argument("--api", action='store_true', help=argparse.SUPPRESS, default=True) group_compat.add_argument("--api-auth", type=str, help=argparse.SUPPRESS, default=None) + group_compat.add_argument("--api-log", default=os.environ.get("SD_APILOG", True), action='store_true', help=argparse.SUPPRESS) + group_compat.add_argument("--disable-queue", default=os.environ.get("SD_DISABLEQUEUE", False), action='store_true', help=argparse.SUPPRESS) + group_compat.add_argument('--debug', default=os.environ.get("SD_DEBUG", True), action='store_true', help=argparse.SUPPRESS) + def settings_args(opts, args): - group_compat = parser.add_argument_group('Compatibility options') # removed args are added here as hidden in fixed format for compatbility reasons + group_compat = parser.add_argument_group('Compatibility options') group_compat.add_argument("--allow-code", default=os.environ.get("SD_ALLOWCODE", False), action='store_true', help=argparse.SUPPRESS) group_compat.add_argument("--use-cpu", nargs='+', default=[], type=str.lower, help=argparse.SUPPRESS) group_compat.add_argument("-f", action='store_true', help=argparse.SUPPRESS) # allows running as root; implemented outside of webui diff --git a/modules/extensions.py b/modules/extensions.py index c2e8dceb7..78f26c655 100644 --- a/modules/extensions.py +++ b/modules/extensions.py @@ -154,4 +154,4 @@ def list_extensions(): for dirname, path, is_builtin in extension_paths: extension = Extension(name=dirname, path=path, enabled=dirname not in disabled_extensions, is_builtin=is_builtin) extensions.append(extension) - shared.log.debug(f'Extensions disabled: {[e.name for e in extensions if not e.enabled]}') + shared.log.debug(f'Extensions: disabled={[e.name for e in extensions if not e.enabled]}') diff --git a/modules/shared.py b/modules/shared.py index 9904309a3..6aa5d9743 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -93,7 +93,6 @@ else: hfcache_dir = os.path.join(os.path.expanduser('~'), '.cache', 'huggingface', 'hub') os.environ["HF_HUB_CACHE"] = hfcache_dir -log.debug(f'Huggingface cache: folder="{hfcache_dir}"') class Backend(Enum): diff --git a/modules/theme.py b/modules/theme.py index 26bb39858..c6450ed29 100644 --- a/modules/theme.py +++ b/modules/theme.py @@ -54,11 +54,9 @@ def list_themes(): huggingface = {x['id'] for x in huggingface if x['status'] == 'RUNNING' and 'test' not in x['id'].lower()} huggingface = [f'huggingface/{x}' for x in huggingface] themes = sorted(gradio) + sorted(huggingface, key=str.casefold) - modules.shared.log.debug(f'UI themes available: type=={modules.shared.opts.theme_type} gradio={len(gradio)} huggingface={len(huggingface)}') elif modules.shared.opts.theme_type == 'Standard': builtin = list_builtin_themes() themes = sorted(builtin) - modules.shared.log.debug(f'UI themes available: type={modules.shared.opts.theme_type} themes={len(builtin)}') elif modules.shared.opts.theme_type == 'Modern': ext = next((e for e in modules.extensions.extensions if e.name == 'sdnext-modernui'), None) if ext is None: @@ -76,7 +74,6 @@ def list_themes(): if len(themes) == 0: themes.append('modern/Default') themes = sorted(themes) - modules.shared.log.debug(f'UI themes available: type={modules.shared.opts.theme_type} themes={len(themes)}') else: modules.shared.log.error(f'UI themes: type={modules.shared.opts.theme_type} unknown') themes = [] @@ -109,11 +106,11 @@ def reload_gradio_theme(): return None elif modules.shared.opts.theme_type == 'Standard': gradio_theme = gr.themes.Base(**default_font_params) - modules.shared.log.info(f'UI theme: type={modules.shared.opts.theme_type} name="{theme_name}"') + modules.shared.log.info(f'UI theme: type={modules.shared.opts.theme_type} name="{theme_name}" available={len(available_themes)}') return 'sdnext.css' elif modules.shared.opts.theme_type == 'Modern': gradio_theme = gr.themes.Base(**default_font_params) - modules.shared.log.info(f'UI theme: type={modules.shared.opts.theme_type} name="{theme_name}"') + modules.shared.log.info(f'UI theme: type={modules.shared.opts.theme_type} name="{theme_name}" available={len(available_themes)}') return 'base.css' elif modules.shared.opts.theme_type == 'None': if theme_name.startswith('gradio/'): diff --git a/scripts/k_diff.py b/scripts/k_diff.py index 92b43149d..117df9c31 100644 --- a/scripts/k_diff.py +++ b/scripts/k_diff.py @@ -1,6 +1,5 @@ import inspect import gradio as gr -import diffusers from modules import scripts, processing, shared, sd_models @@ -38,6 +37,7 @@ def run(self, p: processing.StableDiffusionProcessing, sampler: str): # pylint: if shared.sd_model_type not in self.supported_models: shared.log.warning(f'K-Diffusion: class={shared.sd_model.__class__.__name__} model={shared.sd_model_type} required={self.supported_models}') return None + import diffusers cls = None if shared.sd_model_type == "sd": cls = diffusers.pipelines.StableDiffusionKDiffusionPipeline diff --git a/webui.py b/webui.py index d3a0328eb..0bf8bc199 100644 --- a/webui.py +++ b/webui.py @@ -84,6 +84,8 @@ def initialize(): modules.hashes.init_cache() check_rollback_vae() + log.debug(f'Huggingface cache: path="{shared.opts.hfcache_dir}"') + modules.sd_samplers.list_samplers() timer.startup.record("samplers") @@ -226,7 +228,7 @@ def start_common(): if shared.cmd_opts.data_dir is not None and len(shared.cmd_opts.data_dir) > 0: log.info(f'Using data path: {shared.cmd_opts.data_dir}') if shared.cmd_opts.models_dir is not None and len(shared.cmd_opts.models_dir) > 0 and shared.cmd_opts.models_dir != 'models': - log.info(f'Using models path: {shared.cmd_opts.models_dir}') + log.info(f'Models path: {shared.cmd_opts.models_dir}') create_paths(shared.opts) async_policy() initialize() @@ -321,8 +323,10 @@ def start_ui(): modules.script_callbacks.app_started_callback(shared.demo, app) timer.startup.record("app-started") - time_setup = [f'{k}:{round(v,3)}' for (k,v) in modules.scripts.time_setup.items() if v > 0.005] - shared.log.debug(f'Scripts setup: {time_setup}') + time_sorted = sorted(modules.scripts.time_setup.items(), key=lambda x: x[1], reverse=True) + time_script = [f'{k}:{round(v,3)}' for (k,v) in time_sorted if v > 0.01] + time_total = sum(modules.scripts.time_setup.values()) + shared.log.debug(f'Scripts setup: time={time_total:.3f} {time_script}') time_component = [f'{k}:{round(v,3)}' for (k,v) in modules.scripts.time_component.items() if v > 0.005] if len(time_component) > 0: shared.log.debug(f'Scripts components: {time_component}') From cde11b48b434503d849d9ba084e3e28ed3544535 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 30 Dec 2024 21:31:35 -0500 Subject: [PATCH 193/249] fix hires and corrections with batch processing Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 8 ++++-- modules/processing_class.py | 13 +++++---- modules/processing_correction.py | 48 ++++++++++++-------------------- modules/processing_diffusers.py | 8 +++--- scripts/freescale.py | 12 ++++---- webui.py | 2 +- 6 files changed, 42 insertions(+), 49 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 832ae2dc1..2d4fcf5e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ # Change Log for SD.Next -## Update for 2024-12-30 +## Update for 2024-12-31 + +NYE refresh release with quite a few optimizatios and bug fixes... - **LoRA**: - **Sana** support @@ -40,7 +42,9 @@ - do not show disabled networks - enable debug logging by default - image width/height calculation when doing img2img - + - corrections with batch processing + - hires with refiner prompt and batch processing + ## Update for 2024-12-24 ### Highlights for 2024-12-24 diff --git a/modules/processing_class.py b/modules/processing_class.py index 1f8382128..a8ffd3392 100644 --- a/modules/processing_class.py +++ b/modules/processing_class.py @@ -116,16 +116,18 @@ def __init__(self, # overrides override_settings: Dict[str, Any] = {}, override_settings_restore_afterwards: bool = True, + task_args: Dict[str, Any] = {}, + ops: List[str] = [], # metadata extra_generation_params: Dict[Any, Any] = {}, ): # extra args set by processing loop - self.task_args = {} + self.task_args = task_args # state items self.state: str = '' - self.ops = [] + self.ops = ops self.skip = [] self.color_corrections = [] self.is_control = False @@ -266,7 +268,6 @@ def __init__(self, self.s_max = shared.opts.s_max self.s_tmin = shared.opts.s_tmin self.s_tmax = float('inf') # not representable as a standard ui option - self.task_args = {} # ip adapter self.ip_adapter_names = [] @@ -299,6 +300,9 @@ def __init__(self, self.negative_embeds = [] self.negative_pooleds = [] + def __str__(self): + return f'{self.__class__.__name__}: {self.__dict__}' + @property def sd_model(self): return shared.sd_model @@ -339,9 +343,6 @@ def sample(self, conditioning, unconditional_conditioning, seeds, subseeds, subs def close(self): self.sampler = None # pylint: disable=attribute-defined-outside-init - def __str__(self): - return f'{self.__class__.__name__}: {self.__dict__}' - class StableDiffusionProcessingTxt2Img(StableDiffusionProcessing): def __init__(self, **kwargs): diff --git a/modules/processing_correction.py b/modules/processing_correction.py index 050fae889..73d83d3cd 100644 --- a/modules/processing_correction.py +++ b/modules/processing_correction.py @@ -16,7 +16,7 @@ def sharpen_tensor(tensor, ratio=0): if ratio == 0: - debug("Sharpen: Early exit") + # debug("Sharpen: Early exit") return tensor kernel = torch.ones((3, 3), dtype=tensor.dtype, device=tensor.device) kernel[1, 1] = 5.0 @@ -42,18 +42,18 @@ def soft_clamp_tensor(tensor, threshold=0.8, boundary=4): min_replace = ((tensor + threshold) / (min_vals + threshold)) * (-boundary + threshold) - threshold under_mask = tensor < -threshold tensor = torch.where(over_mask, max_replace, torch.where(under_mask, min_replace, tensor)) - debug(f'HDR soft clamp: threshold={threshold} boundary={boundary} shape={tensor.shape}') + # debug(f'HDR soft clamp: threshold={threshold} boundary={boundary} shape={tensor.shape}') return tensor def center_tensor(tensor, channel_shift=0.0, full_shift=0.0, offset=0.0): if channel_shift == 0 and full_shift == 0 and offset == 0: return tensor - debug(f'HDR center: Before Adjustment: Full mean={tensor.mean().item()} Channel means={tensor.mean(dim=(-1, -2)).float().cpu().numpy()}') + # debug(f'HDR center: Before Adjustment: Full mean={tensor.mean().item()} Channel means={tensor.mean(dim=(-1, -2)).float().cpu().numpy()}') tensor -= tensor.mean(dim=(-1, -2), keepdim=True) * channel_shift tensor -= tensor.mean() * full_shift - offset - debug(f'HDR center: channel-shift={channel_shift} full-shift={full_shift}') - debug(f'HDR center: After Adjustment: Full mean={tensor.mean().item()} Channel means={tensor.mean(dim=(-1, -2)).float().cpu().numpy()}') + # debug(f'HDR center: channel-shift={channel_shift} full-shift={full_shift}') + # debug(f'HDR center: After Adjustment: Full mean={tensor.mean().item()} Channel means={tensor.mean(dim=(-1, -2)).float().cpu().numpy()}') return tensor @@ -65,7 +65,7 @@ def maximize_tensor(tensor, boundary=1.0): max_val = tensor.max() normalization_factor = boundary / max(abs(min_val), abs(max_val)) tensor *= normalization_factor - debug(f'HDR maximize: boundary={boundary} min={min_val} max={max_val} factor={normalization_factor}') + # debug(f'HDR maximize: boundary={boundary} min={min_val} max={max_val} factor={normalization_factor}') return tensor @@ -78,7 +78,7 @@ def get_color(colorstr): def color_adjust(tensor, colorstr, ratio): color = get_color(colorstr) - debug(f'HDR tint: str={colorstr} color={color} ratio={ratio}') + # debug(f'HDR tint: str={colorstr} color={color} ratio={ratio}') for i in range(3): tensor[i] = center_tensor(tensor[i], full_shift=1, offset=color[i]*(ratio/2)) return tensor @@ -86,35 +86,26 @@ def color_adjust(tensor, colorstr, ratio): def correction(p, timestep, latent): if timestep > 950 and p.hdr_clamp: - p.extra_generation_params["HDR clamp"] = f'{p.hdr_threshold}/{p.hdr_boundary}' latent = soft_clamp_tensor(latent, threshold=p.hdr_threshold, boundary=p.hdr_boundary) - if 600 < timestep < 900 and (p.hdr_color != 0 or p.hdr_tint_ratio != 0): - if p.hdr_brightness != 0: - latent[0:1] = center_tensor(latent[0:1], full_shift=float(p.hdr_mode), offset=2*p.hdr_brightness) # Brightness - p.extra_generation_params["HDR brightness"] = f'{p.hdr_brightness}' - p.hdr_brightness = 0 - if p.hdr_color != 0: - latent[1:] = center_tensor(latent[1:], channel_shift=p.hdr_color, full_shift=float(p.hdr_mode)) # Color - p.extra_generation_params["HDR color"] = f'{p.hdr_color}' - p.hdr_color = 0 - if p.hdr_tint_ratio != 0: - latent = color_adjust(latent, p.hdr_color_picker, p.hdr_tint_ratio) - p.hdr_tint_ratio = 0 + p.extra_generation_params["HDR clamp"] = f'{p.hdr_threshold}/{p.hdr_boundary}' + if 600 < timestep < 900 and p.hdr_color != 0: + latent[1:] = center_tensor(latent[1:], channel_shift=p.hdr_color, full_shift=float(p.hdr_mode)) # Color + p.extra_generation_params["HDR color"] = f'{p.hdr_color}' + if 600 < timestep < 900 and p.hdr_tint_ratio != 0: + latent = color_adjust(latent, p.hdr_color_picker, p.hdr_tint_ratio) + p.extra_generation_params["HDR tint"] = f'{p.hdr_tint_ratio}' if timestep < 200 and (p.hdr_brightness != 0): # do it late so it doesn't change the composition - if p.hdr_brightness != 0: - latent[0:1] = center_tensor(latent[0:1], full_shift=float(p.hdr_mode), offset=2*p.hdr_brightness) # Brightness - p.extra_generation_params["HDR brightness"] = f'{p.hdr_brightness}' - p.hdr_brightness = 0 + latent[0:1] = center_tensor(latent[0:1], full_shift=float(p.hdr_mode), offset=p.hdr_brightness) # Brightness + p.extra_generation_params["HDR brightness"] = f'{p.hdr_brightness}' if timestep < 350 and p.hdr_sharpen != 0: - p.extra_generation_params["HDR sharpen"] = f'{p.hdr_sharpen}' per_step_ratio = 2 ** (timestep / 250) * p.hdr_sharpen / 16 if abs(per_step_ratio) > 0.01: - debug(f"HDR Sharpen: timestep={timestep} ratio={p.hdr_sharpen} val={per_step_ratio}") latent = sharpen_tensor(latent, ratio=per_step_ratio) + p.extra_generation_params["HDR sharpen"] = f'{p.hdr_sharpen}' if 1 < timestep < 100 and p.hdr_maximize: - p.extra_generation_params["HDR max"] = f'{p.hdr_max_center}/{p.hdr_max_boundry}' latent = center_tensor(latent, channel_shift=p.hdr_max_center, full_shift=1.0) latent = maximize_tensor(latent, boundary=p.hdr_max_boundry) + p.extra_generation_params["HDR max"] = f'{p.hdr_max_center}/{p.hdr_max_boundry}' return latent @@ -129,9 +120,6 @@ def correction_callback(p, timestep, kwargs, initial: bool = False): elif skip_correction: return kwargs latents = kwargs["latents"] - if debug_enabled: - debug('') - debug(f' Timestep: {timestep}') # debug(f'HDR correction: latents={latents.shape}') if len(latents.shape) == 4: # standard batched latent for i in range(latents.shape[0]): diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 4f8125d4f..bd759e5f0 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -205,10 +205,10 @@ def process_hires(p: processing.StableDiffusionProcessing, output): hires_args = set_pipeline_args( p=p, model=shared.sd_model, - prompts=[p.refiner_prompt] if len(p.refiner_prompt) > 0 else p.prompts, - negative_prompts=[p.refiner_negative] if len(p.refiner_negative) > 0 else p.negative_prompts, - prompts_2=[p.refiner_prompt] if len(p.refiner_prompt) > 0 else p.prompts, - negative_prompts_2=[p.refiner_negative] if len(p.refiner_negative) > 0 else p.negative_prompts, + prompts=len(output.images)* [p.refiner_prompt] if len(p.refiner_prompt) > 0 else p.prompts, + negative_prompts=len(output.images) * [p.refiner_negative] if len(p.refiner_negative) > 0 else p.negative_prompts, + prompts_2=len(output.images) * [p.refiner_prompt] if len(p.refiner_prompt) > 0 else p.prompts, + negative_prompts_2=len(output.images) * [p.refiner_negative] if len(p.refiner_negative) > 0 else p.negative_prompts, num_inference_steps=calculate_hires_steps(p), eta=shared.opts.scheduler_eta, guidance_scale=p.image_cfg_scale if p.image_cfg_scale is not None else p.cfg_scale, diff --git a/scripts/freescale.py b/scripts/freescale.py index 672ceea41..31c5a3b1c 100644 --- a/scripts/freescale.py +++ b/scripts/freescale.py @@ -35,16 +35,16 @@ def ui(self, _is_img2img): # ui elements s1_restart = gr.Slider(minimum=0, maximum=1.0, value=0.75, label='Restart step') with gr.Row(): s2_enable = gr.Checkbox(value=True, label='2nd Stage') - s2_scale = gr.Slider(minimum=1, maximum=8.0, value=2.0, label='Scale') - s2_restart = gr.Slider(minimum=0, maximum=1.0, value=0.75, label='Restart step') + s2_scale = gr.Slider(minimum=1, maximum=8.0, value=2.0, label='2nd Scale') + s2_restart = gr.Slider(minimum=0, maximum=1.0, value=0.75, label='2nd Restart step') with gr.Row(): s3_enable = gr.Checkbox(value=False, label='3rd Stage') - s3_scale = gr.Slider(minimum=1, maximum=8.0, value=3.0, label='Scale') - s3_restart = gr.Slider(minimum=0, maximum=1.0, value=0.75, label='Restart step') + s3_scale = gr.Slider(minimum=1, maximum=8.0, value=3.0, label='3rd Scale') + s3_restart = gr.Slider(minimum=0, maximum=1.0, value=0.75, label='3rd Restart step') with gr.Row(): s4_enable = gr.Checkbox(value=False, label='4th Stage') - s4_scale = gr.Slider(minimum=1, maximum=8.0, value=4.0, label='Scale') - s4_restart = gr.Slider(minimum=0, maximum=1.0, value=0.75, label='Restart step') + s4_scale = gr.Slider(minimum=1, maximum=8.0, value=4.0, label='4th Scale') + s4_restart = gr.Slider(minimum=0, maximum=1.0, value=0.75, label='4th Restart step') return [cosine_scale, override_sampler, cosine_scale_bg, dilate_tau, s1_enable, s1_scale, s1_restart, s2_enable, s2_scale, s2_restart, s3_enable, s3_scale, s3_restart, s4_enable, s4_scale, s4_restart] def run(self, p: processing.StableDiffusionProcessing, cosine_scale, override_sampler, cosine_scale_bg, dilate_tau, s1_enable, s1_scale, s1_restart, s2_enable, s2_scale, s2_restart, s3_enable, s3_scale, s3_restart, s4_enable, s4_scale, s4_restart): # pylint: disable=arguments-differ diff --git a/webui.py b/webui.py index 0bf8bc199..9c85fe426 100644 --- a/webui.py +++ b/webui.py @@ -303,7 +303,7 @@ def start_ui(): shared.log.info(f'API ReDocs: {local_url[:-1]}/redocs') # pylint: disable=unsubscriptable-object if share_url is not None: shared.log.info(f'Share URL: {share_url}') - shared.log.debug(f'Gradio functions: registered={len(shared.demo.fns)}') + # shared.log.debug(f'Gradio functions: registered={len(shared.demo.fns)}') shared.demo.server.wants_restart = False setup_middleware(app, cmd_opts) From f0942d5b7a740cb3340affab9c3a55a889316de7 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 30 Dec 2024 21:48:26 -0500 Subject: [PATCH 194/249] log cleanup Signed-off-by: Vladimir Mandic --- modules/devices.py | 2 +- modules/images_resize.py | 2 +- modules/processing_args.py | 3 ++- modules/processing_vae.py | 4 ++-- modules/ui_extra_networks.py | 2 +- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/modules/devices.py b/modules/devices.py index 6168ac63a..c98aea696 100644 --- a/modules/devices.py +++ b/modules/devices.py @@ -486,7 +486,7 @@ def set_cuda_params(): device_name = get_raw_openvino_device() else: device_name = torch.device(get_optimal_device_name()) - log.info(f'Torch parameters: backend={backend} device={device_name} config={opts.cuda_dtype} dtype={dtype} vae={dtype_vae} unet={dtype_unet} context={inference_context.__name__} nohalf={opts.no_half} nohalfvae={opts.no_half_vae} upcast={opts.upcast_sampling} deterministic={opts.cudnn_deterministic} test-fp16={fp16_ok} test-bf16={bf16_ok} optimization="{opts.cross_attention_optimization}"') + log.info(f'Torch parameters: backend={backend} device={device_name} config={opts.cuda_dtype} dtype={dtype} context={inference_context.__name__} nohalf={opts.no_half} nohalfvae={opts.no_half_vae} upcast={opts.upcast_sampling} deterministic={opts.cudnn_deterministic} fp16={"pass" if fp16_ok else "fail"} bf16={"pass" if bf16_ok else "fail"} optimization="{opts.cross_attention_optimization}"') def cond_cast_unet(tensor): diff --git a/modules/images_resize.py b/modules/images_resize.py index 362be79ee..183e1d7f1 100644 --- a/modules/images_resize.py +++ b/modules/images_resize.py @@ -129,5 +129,5 @@ def context_aware(im, width, height, context): shared.log.error(f'Invalid resize mode: {resize_mode}') t1 = time.time() fn = f'{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access - shared.log.debug(f'Image resize: input={im} width={width} height={height} mode="{shared.resize_modes[resize_mode]}" upscaler="{upscaler_name}" context="{context}" type={output_type} result={res} time={t1-t0:.2f} fn={fn}') # pylint: disable=protected-access + shared.log.debug(f'Image resize: source={im.width}:{im.height} target={width}:{height} mode="{shared.resize_modes[resize_mode]}" upscaler="{upscaler_name}" type={output_type} time={t1-t0:.2f} fn={fn}') # pylint: disable=protected-access return np.array(res) if output_type == 'np' else res diff --git a/modules/processing_args.py b/modules/processing_args.py index 34e93e8d4..f39537538 100644 --- a/modules/processing_args.py +++ b/modules/processing_args.py @@ -340,7 +340,8 @@ def set_pipeline_args(p, model, prompts:list, negative_prompts:list, prompts_2:t if not debug_enabled and k.endswith('_embeds'): del clean[k] clean['prompt'] = 'embeds' - shared.log.info(f'{desc}: pipeline={model.__class__.__name__} task={sd_models.get_diffusers_task(model)} batch={p.iteration + 1}/{p.n_iter}x{p.batch_size} set={clean}') + task = str(sd_models.get_diffusers_task(model)).replace('DiffusersTaskType.', '') + shared.log.info(f'{desc}: pipeline={model.__class__.__name__} task={task} batch={p.iteration + 1}/{p.n_iter}x{p.batch_size} set={clean}') if p.hdr_clamp or p.hdr_maximize or p.hdr_brightness != 0 or p.hdr_color != 0 or p.hdr_sharpen != 0: shared.log.debug(f'HDR: clamp={p.hdr_clamp} maximize={p.hdr_maximize} brightness={p.hdr_brightness} color={p.hdr_color} sharpen={p.hdr_sharpen} threshold={p.hdr_threshold} boundary={p.hdr_boundary} max={p.hdr_max_boundry} center={p.hdr_max_center}') diff --git a/modules/processing_vae.py b/modules/processing_vae.py index 034d17065..04af9bab1 100644 --- a/modules/processing_vae.py +++ b/modules/processing_vae.py @@ -136,8 +136,8 @@ def full_vae_decode(latents, model): vae_name = os.path.splitext(os.path.basename(sd_vae.loaded_vae_file))[0] if sd_vae.loaded_vae_file is not None else "default" vae_stats = f'vae="{vae_name}" dtype={model.vae.dtype} device={model.vae.device} upcast={upcast} slicing={getattr(model.vae, "use_slicing", None)} tiling={getattr(model.vae, "use_tiling", None)}' - latents_stats = f'shape={latents.shape} dtype={latents.dtype} device={latents.device}' - stats = f'{vae_stats} latents {latents_stats}' + latents_stats = f'latents={latents.shape}:{latents.device}:{latents.dtype}' + stats = f'{vae_stats} {latents_stats}' log_debug(f'VAE config: {model.vae.config}') try: diff --git a/modules/ui_extra_networks.py b/modules/ui_extra_networks.py index 3594939b2..769f47581 100644 --- a/modules/ui_extra_networks.py +++ b/modules/ui_extra_networks.py @@ -277,7 +277,7 @@ def create_page(self, tabname, skip = False): self.html += ''.join(htmls) self.page_time = time.time() self.html = f"
{subdirs_html}
{self.html}
" - shared.log.debug(f"Networks: page='{self.name}' items={len(self.items)} subfolders={len(subdirs)} tab={tabname} folders={self.allowed_directories_for_previews()} list={self.list_time:.2f} thumb={self.preview_time:.2f} desc={self.desc_time:.2f} info={self.info_time:.2f} workers={shared.max_workers} sort={shared.opts.extra_networks_sort}") + shared.log.debug(f"Networks: page='{self.name}' items={len(self.items)} subfolders={len(subdirs)} tab={tabname} folders={self.allowed_directories_for_previews()} list={self.list_time:.2f} thumb={self.preview_time:.2f} desc={self.desc_time:.2f} info={self.info_time:.2f} workers={shared.max_workers}") if len(self.missing_thumbs) > 0: threading.Thread(target=self.create_thumb).start() return self.patch(self.html, tabname) From 6ad9579890b4fe989563d00b202e81b1d836dded Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 31 Dec 2024 08:00:29 -0500 Subject: [PATCH 195/249] wiki update Signed-off-by: Vladimir Mandic --- wiki | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wiki b/wiki index 04166d074..7d159b39f 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 04166d0741dfb9b882884be2ac614b2b720b5e07 +Subproject commit 7d159b39f70b1e6cc5673b39180b50e6688450f3 From cde4585ff972d4fc907c6ac834bef5afd3295c3a Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 31 Dec 2024 09:09:44 -0500 Subject: [PATCH 196/249] fix processing with nested calls Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 3 ++- installer.py | 3 ++- modules/processing_class.py | 16 ++++++++++------ 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d4fcf5e5..c45af644d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,7 +43,8 @@ NYE refresh release with quite a few optimizatios and bug fixes... - enable debug logging by default - image width/height calculation when doing img2img - corrections with batch processing - - hires with refiner prompt and batch processing + - hires with refiner prompt and batch processing + - processing with nested calls ## Update for 2024-12-24 diff --git a/installer.py b/installer.py index eb6eb8511..60aab0ddd 100644 --- a/installer.py +++ b/installer.py @@ -54,7 +54,7 @@ class Dot(dict): # dot notation access to dictionary attributes diffusers_commit = "unknown" extensions_commit = { 'sd-webui-controlnet': 'ecd33eb', - 'adetailer': 'a89c01d' + # 'adetailer': 'a89c01d' # 'stable-diffusion-webui-images-browser': '27fe4a7', } @@ -1075,6 +1075,7 @@ def set_environment(): os.environ.setdefault('TF_CPP_MIN_LOG_LEVEL', '2') os.environ.setdefault('TF_ENABLE_ONEDNN_OPTS', '0') os.environ.setdefault('USE_TORCH', '1') + os.environ.setdefault('TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD', '1') os.environ.setdefault('UVICORN_TIMEOUT_KEEP_ALIVE', '60') os.environ.setdefault('KINETO_LOG_LEVEL', '3') os.environ.setdefault('DO_NOT_TRACK', '1') diff --git a/modules/processing_class.py b/modules/processing_class.py index a8ffd3392..96761d262 100644 --- a/modules/processing_class.py +++ b/modules/processing_class.py @@ -116,18 +116,23 @@ def __init__(self, # overrides override_settings: Dict[str, Any] = {}, override_settings_restore_afterwards: bool = True, - task_args: Dict[str, Any] = {}, - ops: List[str] = [], # metadata - extra_generation_params: Dict[Any, Any] = {}, + # extra_generation_params: Dict[Any, Any] = {}, + # task_args: Dict[str, Any] = {}, + # ops: List[str] = [], + **kwargs, ): + for k, v in kwargs.items(): + setattr(self, k, v) + # extra args set by processing loop - self.task_args = task_args + self.task_args = {} + self.extra_generation_params = {} # state items self.state: str = '' - self.ops = ops + self.ops = [] self.skip = [] self.color_corrections = [] self.is_control = False @@ -203,7 +208,6 @@ def __init__(self, self.do_not_save_samples = do_not_save_samples self.do_not_save_grid = do_not_save_grid self.override_settings_restore_afterwards = override_settings_restore_afterwards - self.extra_generation_params = extra_generation_params self.eta = eta self.cfg_scale = cfg_scale self.cfg_end = cfg_end From 5c36d7b84772de1889273d38972ebbc5dbe59a60 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 31 Dec 2024 09:15:31 -0500 Subject: [PATCH 197/249] torch 2.6.0 support Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 22 ++++++++++++---------- modules/masking.py | 12 ++++++------ 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c45af644d..d9b2b0aed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,31 +4,33 @@ NYE refresh release with quite a few optimizatios and bug fixes... -- **LoRA**: +- **LoRA**: - **Sana** support - quantized models support - fuse support with on-demand apply/unapply - add legacy option in *settings -> networks* -- **HunyuanVideo** +- **HunyuanVideo**: - optimizations: full offload, quantization and tiling support -- **LTXVideo** +- **LTXVideo**: - optimizations: full offload, quantization and tiling support - [TeaCache](https://github.com/ali-vilab/TeaCache/blob/main/TeaCache4LTX-Video/README.md) integration -- **VAE**: +- **VAE**: - tiling granular options in *settings -> variable auto encoder* -- **UI**: +- **UI**: - live preview optimizations and error handling - live preview high quality output, thanks @Disty0 - CSS optimizations when log view is disabled -- **Samplers**: +- **Samplers**: - add flow shift options and separate dynamic thresholding from dynamic shifting - autodetect matching sigma capabilities -- **API** +- **API**: - better default values for generate -- **Refactor**: +- **Refactor**: - remove all LDM imports if running in native mode - - startup optimizatios -- **Fixes** + - startup optimizatios +- **Torch**: + - support for `torch==2.6.0` +- **Fixes**: - flux pipeline switches: txt/img/inpaint - flux custom unet loader for bnb - interrogate caption with T5 diff --git a/modules/masking.py b/modules/masking.py index 42e8e894e..51198d979 100644 --- a/modules/masking.py +++ b/modules/masking.py @@ -388,7 +388,7 @@ def run_mask(input_image: Image.Image, input_mask: Image.Image = None, return_ty if input_image is None: return input_mask - t0 = time.time() + # t0 = time.time() input_mask = get_mask(input_image, input_mask) # perform optional auto-masking if input_mask is None: return None @@ -436,14 +436,14 @@ def run_mask(input_image: Image.Image, input_mask: Image.Image = None, return_ty if opts.invert: mask = np.invert(mask) - mask_size = np.count_nonzero(mask) - total_size = np.prod(mask.shape) - area_size = np.count_nonzero(mask) - t1 = time.time() return_type = return_type or opts.preview_type - shared.log.debug(f'Mask: size={input_image.width}x{input_image.height} masked={mask_size}px area={area_size/total_size:.2f} auto={opts.auto_mask} blur={opts.mask_blur:.3f} erode={opts.mask_erode:.3f} dilate={opts.mask_dilate:.3f} type={return_type} time={t1-t0:.2f}') + # mask_size = np.count_nonzero(mask) + # total_size = np.prod(mask.shape) + # area_size = np.count_nonzero(mask) + # t1 = time.time() + # shared.log.debug(f'Mask: size={input_image.width}x{input_image.height} masked={mask_size}px area={area_size/total_size:.2f} auto={opts.auto_mask} blur={opts.mask_blur:.3f} erode={opts.mask_erode:.3f} dilate={opts.mask_dilate:.3f} type={return_type} time={t1-t0:.2f}') if return_type == 'None': return input_mask elif return_type == 'Opaque': From 5c07b942d06fd99f17e7f4e71093d9fbfafce5eb Mon Sep 17 00:00:00 2001 From: Disty0 Date: Tue, 31 Dec 2024 17:45:23 +0300 Subject: [PATCH 198/249] OpenVINO fix shapes resolution change and disable re-compile --- CHANGELOG.md | 3 + modules/intel/openvino/__init__.py | 103 +++++++++++++++++++---------- modules/sd_models_compile.py | 60 ++++++++++------- 3 files changed, 107 insertions(+), 59 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d9b2b0aed..0559b184a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,9 @@ NYE refresh release with quite a few optimizatios and bug fixes... - startup optimizatios - **Torch**: - support for `torch==2.6.0` +- **OpenVINO**: + - disable re-compile on resolution change + - fix shape mismatch on resolution change - **Fixes**: - flux pipeline switches: txt/img/inpaint - flux custom unet loader for bnb diff --git a/modules/intel/openvino/__init__.py b/modules/intel/openvino/__init__.py index 157d26d96..f441e6907 100644 --- a/modules/intel/openvino/__init__.py +++ b/modules/intel/openvino/__init__.py @@ -32,16 +32,24 @@ class OpenVINOGraphModule(torch.nn.Module): - def __init__(self, gm, partition_id, use_python_fusion_cache, model_hash_str: str = None, file_name=""): + def __init__(self, gm, partition_id, use_python_fusion_cache, model_hash_str: str = None, file_name="", signature="", int_inputs=[]): super().__init__() self.gm = gm + self.int_inputs = int_inputs self.partition_id = partition_id + self.signature = signature self.executor_parameters = {"use_python_fusion_cache": use_python_fusion_cache, "model_hash_str": model_hash_str} self.file_name = file_name def __call__(self, *args): - result = openvino_execute(self.gm, *args, executor_parameters=self.executor_parameters, partition_id=self.partition_id, file_name=self.file_name) + ov_inputs = [] + for arg in args: + if not isinstance(arg, int): + ov_inputs.append(arg) + for idx, int_input in self.int_inputs: + ov_inputs.insert(idx, int_input) + result = openvino_execute(self.gm, *ov_inputs, executor_parameters=self.executor_parameters, partition_id=self.partition_id, file_name=self.file_name, signature=self.signature) return result @@ -111,10 +119,7 @@ def cached_model_name(model_hash_str, device, args, cache_root, reversed = False else: inputs_str += "_" + "torch.SymInt1" elif isinstance(input_data, int): - if reversed: - inputs_str = "_" + "int" + inputs_str - else: - inputs_str += "_" + "int" + pass else: if reversed: inputs_str = "_" + str(input_data.type()) + str(input_data.size())[11:-1].replace(" ", "") + inputs_str @@ -174,16 +179,13 @@ def openvino_compile(gm: GraphModule, *example_inputs, model_hash_str: str = Non input_types.append(torch.SymInt) input_shapes.append(torch.Size([1])) elif isinstance(input_data, int): - input_types.append(torch.int64) - input_shapes.append(torch.Size([1])) + pass else: input_types.append(input_data.type()) input_shapes.append(input_data.size()) decoder = TorchFXPythonDecoder(gm, input_shapes=input_shapes, input_types=input_types) - im = fe.load(decoder) - om = fe.convert(im) if file_name is not None: @@ -206,13 +208,13 @@ def openvino_compile(gm: GraphModule, *example_inputs, model_hash_str: str = Non torch.bool: Type.boolean } + idx_minus = 0 for idx, input_data in enumerate(example_inputs): if isinstance(input_data, int): - om.inputs[idx].get_node().set_element_type(dtype_mapping[torch.int64]) - om.inputs[idx].get_node().set_partial_shape(PartialShape(list(torch.Size([1])))) + idx_minus += 1 else: - om.inputs[idx].get_node().set_element_type(dtype_mapping[input_data.dtype]) - om.inputs[idx].get_node().set_partial_shape(PartialShape(list(input_data.shape))) + om.inputs[idx-idx_minus].get_node().set_element_type(dtype_mapping[input_data.dtype]) + om.inputs[idx-idx_minus].get_node().set_partial_shape(PartialShape(list(input_data.shape))) om.validate_nodes_and_infer_types() if shared.opts.nncf_quantize and not dont_use_quant: @@ -305,35 +307,49 @@ def openvino_compile_cached_model(cached_model_path, *example_inputs): return compiled_model -def openvino_execute(gm: GraphModule, *args, executor_parameters=None, partition_id, file_name=""): +def openvino_execute(gm: GraphModule, *args, executor_parameters=None, partition_id=None, file_name="", signature=""): + if isinstance(gm, OpenVINOGraphModule): + partition_id = gm.partition_id + file_name = gm.file_name + signature = gm.signature + executor_parameters = gm.executor_parameters + gm = gm.gm executor_parameters = executor_parameters or DEFAULT_OPENVINO_PYTHON_CONFIG - use_cache = executor_parameters.get( + if partition_id is not None: + model_cache_str = str(partition_id) + "_" + signature + elif signature: + model_cache_str = signature + + use_cache = model_cache_str and executor_parameters.get( "use_python_fusion_cache", DEFAULT_OPENVINO_PYTHON_CONFIG["use_python_fusion_cache"], ) model_hash_str = executor_parameters.get("model_hash_str", None) if model_hash_str is not None: - model_hash_str = model_hash_str + str(partition_id) + model_hash_str = model_hash_str + str(partition_id) if partition_id is not None else "" - if use_cache and (partition_id in shared.compiled_model_state.compiled_cache): - compiled = shared.compiled_model_state.compiled_cache[partition_id] - req = shared.compiled_model_state.req_cache[partition_id] + if use_cache and (model_cache_str in shared.compiled_model_state.compiled_cache): + compiled = shared.compiled_model_state.compiled_cache[model_cache_str] + req = shared.compiled_model_state.req_cache[model_cache_str] else: if (shared.compiled_model_state.cn_model != [] and file_name is not None and os.path.isfile(file_name + ".xml") and os.path.isfile(file_name + ".bin")): compiled = openvino_compile_cached_model(file_name, *args) else: compiled = openvino_compile(gm, *args, model_hash_str=model_hash_str, file_name=file_name) - shared.compiled_model_state.compiled_cache[partition_id] = compiled + if use_cache: + shared.compiled_model_state.compiled_cache[model_cache_str] = compiled req = compiled.create_infer_request() - shared.compiled_model_state.req_cache[partition_id] = req + if use_cache: + shared.compiled_model_state.req_cache[model_cache_str] = req flat_args, _ = tree_flatten(args) ov_inputs = [] for arg in flat_args: - ov_inputs.append((arg if isinstance(arg, int) else arg.detach().cpu().numpy())) + if not isinstance(arg, int): + ov_inputs.append((arg.detach().cpu().numpy())) res = req.infer(ov_inputs, share_inputs=True, share_outputs=True) @@ -352,33 +368,50 @@ def openvino_execute_partitioned(gm: GraphModule, *args, executor_parameters=Non ) model_hash_str = executor_parameters.get("model_hash_str", None) - signature = str(id(gm)) + signature = "signature" #str(id(gm)) + idx_minus = 0 + int_inputs = [] for idx, input_data in enumerate(args): - if isinstance(input_data, torch.Tensor): - signature = signature + "_" + str(idx) + ":" + str(input_data.type())[6:] + ":" + str(input_data.size())[11:-1].replace(" ", "") + if isinstance(input_data, int): + int_inputs.append([idx, input_data]) + idx_minus += 1 + elif isinstance(input_data, torch.Tensor): + signature = signature + "_" + str(idx-idx_minus) + ":" + str(input_data.type())[6:] + ":" + str(input_data.size())[11:-1].replace(" ", "") else: - signature = signature + "_" + str(idx) + ":" + type(input_data).__name__ + ":val(" + str(input_data) + ")" + signature = signature + "_" + str(idx-idx_minus) + ":" + type(input_data).__name__ + ":val(" + str(input_data) + ")" if signature not in shared.compiled_model_state.partitioned_modules: - shared.compiled_model_state.partitioned_modules[signature] = partition_graph(gm, use_python_fusion_cache=use_python_fusion_cache, - model_hash_str=model_hash_str, file_name=file_name) + shared.compiled_model_state.partitioned_modules[signature] = partition_graph(gm, use_python_fusion_cache=use_python_fusion_cache, + model_hash_str=model_hash_str, file_name=file_name, signature=signature, int_inputs=int_inputs) - return shared.compiled_model_state.partitioned_modules[signature](*args) + ov_inputs = [] + for arg in args: + if not isinstance(arg, int): + ov_inputs.append(arg) + for idx, int_input in shared.compiled_model_state.partitioned_modules[signature][1]: + ov_inputs.insert(idx, int_input) + return shared.compiled_model_state.partitioned_modules[signature][0](*ov_inputs) -def partition_graph(gm: GraphModule, use_python_fusion_cache: bool, model_hash_str: str = None, file_name=""): +def partition_graph(gm: GraphModule, use_python_fusion_cache: bool, model_hash_str: str = None, file_name="", signature="", int_inputs=[]): for node in gm.graph.nodes: if node.op == "call_module" and "fused_" in node.name: openvino_submodule = getattr(gm, node.name) + if isinstance(openvino_submodule, OpenVINOGraphModule): + openvino_submodule.signature = signature + int_inputs = openvino_submodule.int_inputs + if isinstance(openvino_submodule.gm, OpenVINOGraphModule): + continue gm.delete_submodule(node.target) gm.add_submodule( node.target, - OpenVINOGraphModule(openvino_submodule, shared.compiled_model_state.partition_id, use_python_fusion_cache, - model_hash_str=model_hash_str, file_name=file_name), + OpenVINOGraphModule( + openvino_submodule, shared.compiled_model_state.partition_id, use_python_fusion_cache, + model_hash_str=model_hash_str, file_name=file_name, signature=signature, int_inputs=int_inputs), ) - shared.compiled_model_state.partition_id = shared.compiled_model_state.partition_id + 1 + shared.compiled_model_state.partition_id += 1 - return gm + return gm, int_inputs def generate_subgraph_str(tensor): diff --git a/modules/sd_models_compile.py b/modules/sd_models_compile.py index 20a7d7de2..ebc0460fe 100644 --- a/modules/sd_models_compile.py +++ b/modules/sd_models_compile.py @@ -315,9 +315,9 @@ def optimize_openvino(sd_model): shared.compiled_model_state.partitioned_modules.clear() shared.compiled_model_state = CompiledModelState() shared.compiled_model_state.is_compiled = True - shared.compiled_model_state.first_pass = True if not shared.opts.cuda_compile_precompile else False - shared.compiled_model_state.first_pass_vae = True if not shared.opts.cuda_compile_precompile else False - shared.compiled_model_state.first_pass_refiner = True if not shared.opts.cuda_compile_precompile else False + shared.compiled_model_state.first_pass = not shared.opts.cuda_compile_precompile + shared.compiled_model_state.first_pass_vae = not shared.opts.cuda_compile_precompile + shared.compiled_model_state.first_pass_refiner = not shared.opts.cuda_compile_precompile sd_models.set_accelerate(sd_model) except Exception as e: shared.log.warning(f"Model compile: task=OpenVINO: {e}") @@ -532,30 +532,42 @@ def torchao_model(model, op=None, sd_model=None): # pylint: disable=unused-argum def openvino_recompile_model(p, hires=False, refiner=False): # recompile if a parameter changes - if 'Model' in shared.opts.cuda_compile and shared.opts.cuda_compile_backend != 'none': - if shared.opts.cuda_compile_backend == "openvino_fx": - compile_height = p.height if not hires and hasattr(p, 'height') else p.hr_upscale_to_y - compile_width = p.width if not hires and hasattr(p, 'width') else p.hr_upscale_to_x - if (shared.compiled_model_state is None or - (not shared.compiled_model_state.first_pass - and (shared.compiled_model_state.height != compile_height - or shared.compiled_model_state.width != compile_width - or shared.compiled_model_state.batch_size != p.batch_size))): - if refiner: - shared.log.info("OpenVINO: Recompiling refiner") - sd_models.unload_model_weights(op='refiner') - sd_models.reload_model_weights(op='refiner') - else: - shared.log.info("OpenVINO: Recompiling base model") - sd_models.unload_model_weights(op='model') - sd_models.reload_model_weights(op='model') - shared.compiled_model_state.height = compile_height - shared.compiled_model_state.width = compile_width - shared.compiled_model_state.batch_size = p.batch_size + if shared.opts.cuda_compile_backend == "openvino_fx" and 'Model' in shared.opts.cuda_compile: + compile_height = p.height if not hires and hasattr(p, 'height') else p.hr_upscale_to_y + compile_width = p.width if not hires and hasattr(p, 'width') else p.hr_upscale_to_x + """ + if shared.compiled_model_state is None: + openvino_first_pass = True + else: + if refiner: + openvino_first_pass = shared.compiled_model_state.first_pass_refiner + else: + openvino_first_pass = shared.compiled_model_state.first_pass + if (shared.compiled_model_state is None or + ( + not openvino_first_pass + and ( + shared.compiled_model_state.height != compile_height + or shared.compiled_model_state.width != compile_width + or shared.compiled_model_state.batch_size != p.batch_size + ) + )): + if refiner: + shared.log.info("OpenVINO: Recompiling refiner") + sd_models.unload_model_weights(op='refiner') + sd_models.reload_model_weights(op='refiner') + else: + shared.log.info("OpenVINO: Recompiling base model") + sd_models.unload_model_weights(op='model') + sd_models.reload_model_weights(op='model') + """ + shared.compiled_model_state.height = compile_height + shared.compiled_model_state.width = compile_width + shared.compiled_model_state.batch_size = p.batch_size def openvino_post_compile(op="base"): # delete unet after OpenVINO compile - if 'Model' in shared.opts.cuda_compile and shared.opts.cuda_compile_backend == "openvino_fx": + if shared.opts.cuda_compile_backend == "openvino_fx" and 'Model' in shared.opts.cuda_compile: if shared.compiled_model_state.first_pass and op == "base": shared.compiled_model_state.first_pass = False if not shared.opts.openvino_disable_memory_cleanup and hasattr(shared.sd_model, "unet"): From f0a6df85a45d2d1403b4a3fd1b83b046ddf596ec Mon Sep 17 00:00:00 2001 From: Disty0 Date: Tue, 31 Dec 2024 18:39:34 +0300 Subject: [PATCH 199/249] Cleanup --- modules/intel/openvino/__init__.py | 40 +++++++++++------------------- 1 file changed, 15 insertions(+), 25 deletions(-) diff --git a/modules/intel/openvino/__init__.py b/modules/intel/openvino/__init__.py index f441e6907..88d579a94 100644 --- a/modules/intel/openvino/__init__.py +++ b/modules/intel/openvino/__init__.py @@ -32,12 +32,11 @@ class OpenVINOGraphModule(torch.nn.Module): - def __init__(self, gm, partition_id, use_python_fusion_cache, model_hash_str: str = None, file_name="", signature="", int_inputs=[]): + def __init__(self, gm, partition_id, use_python_fusion_cache, model_hash_str: str = None, file_name="", int_inputs=[]): super().__init__() self.gm = gm self.int_inputs = int_inputs self.partition_id = partition_id - self.signature = signature self.executor_parameters = {"use_python_fusion_cache": use_python_fusion_cache, "model_hash_str": model_hash_str} self.file_name = file_name @@ -49,7 +48,7 @@ def __call__(self, *args): ov_inputs.append(arg) for idx, int_input in self.int_inputs: ov_inputs.insert(idx, int_input) - result = openvino_execute(self.gm, *ov_inputs, executor_parameters=self.executor_parameters, partition_id=self.partition_id, file_name=self.file_name, signature=self.signature) + result = openvino_execute(self.gm, *ov_inputs, executor_parameters=self.executor_parameters, partition_id=self.partition_id, file_name=self.file_name) return result @@ -307,21 +306,14 @@ def openvino_compile_cached_model(cached_model_path, *example_inputs): return compiled_model -def openvino_execute(gm: GraphModule, *args, executor_parameters=None, partition_id=None, file_name="", signature=""): - if isinstance(gm, OpenVINOGraphModule): +def openvino_execute(gm: GraphModule, *args, executor_parameters=None, partition_id=None, file_name=""): + if hasattr(gm, "partition_id"): partition_id = gm.partition_id - file_name = gm.file_name - signature = gm.signature - executor_parameters = gm.executor_parameters + if hasattr(gm, "gm"): gm = gm.gm executor_parameters = executor_parameters or DEFAULT_OPENVINO_PYTHON_CONFIG - if partition_id is not None: - model_cache_str = str(partition_id) + "_" + signature - elif signature: - model_cache_str = signature - - use_cache = model_cache_str and executor_parameters.get( + use_cache = partition_id is not None and executor_parameters.get( "use_python_fusion_cache", DEFAULT_OPENVINO_PYTHON_CONFIG["use_python_fusion_cache"], ) @@ -330,9 +322,9 @@ def openvino_execute(gm: GraphModule, *args, executor_parameters=None, partition if model_hash_str is not None: model_hash_str = model_hash_str + str(partition_id) if partition_id is not None else "" - if use_cache and (model_cache_str in shared.compiled_model_state.compiled_cache): - compiled = shared.compiled_model_state.compiled_cache[model_cache_str] - req = shared.compiled_model_state.req_cache[model_cache_str] + if use_cache and (partition_id in shared.compiled_model_state.compiled_cache.keys()): + compiled = shared.compiled_model_state.compiled_cache[partition_id] + req = shared.compiled_model_state.req_cache[partition_id] else: if (shared.compiled_model_state.cn_model != [] and file_name is not None and os.path.isfile(file_name + ".xml") and os.path.isfile(file_name + ".bin")): @@ -340,10 +332,10 @@ def openvino_execute(gm: GraphModule, *args, executor_parameters=None, partition else: compiled = openvino_compile(gm, *args, model_hash_str=model_hash_str, file_name=file_name) if use_cache: - shared.compiled_model_state.compiled_cache[model_cache_str] = compiled + shared.compiled_model_state.compiled_cache[partition_id] = compiled req = compiled.create_infer_request() if use_cache: - shared.compiled_model_state.req_cache[model_cache_str] = req + shared.compiled_model_state.req_cache[partition_id] = req flat_args, _ = tree_flatten(args) ov_inputs = [] @@ -382,7 +374,7 @@ def openvino_execute_partitioned(gm: GraphModule, *args, executor_parameters=Non if signature not in shared.compiled_model_state.partitioned_modules: shared.compiled_model_state.partitioned_modules[signature] = partition_graph(gm, use_python_fusion_cache=use_python_fusion_cache, - model_hash_str=model_hash_str, file_name=file_name, signature=signature, int_inputs=int_inputs) + model_hash_str=model_hash_str, file_name=file_name, int_inputs=int_inputs) ov_inputs = [] for arg in args: @@ -393,21 +385,19 @@ def openvino_execute_partitioned(gm: GraphModule, *args, executor_parameters=Non return shared.compiled_model_state.partitioned_modules[signature][0](*ov_inputs) -def partition_graph(gm: GraphModule, use_python_fusion_cache: bool, model_hash_str: str = None, file_name="", signature="", int_inputs=[]): +def partition_graph(gm: GraphModule, use_python_fusion_cache: bool, model_hash_str: str = None, file_name="", int_inputs=[]): for node in gm.graph.nodes: if node.op == "call_module" and "fused_" in node.name: openvino_submodule = getattr(gm, node.name) if isinstance(openvino_submodule, OpenVINOGraphModule): - openvino_submodule.signature = signature int_inputs = openvino_submodule.int_inputs - if isinstance(openvino_submodule.gm, OpenVINOGraphModule): - continue + continue gm.delete_submodule(node.target) gm.add_submodule( node.target, OpenVINOGraphModule( openvino_submodule, shared.compiled_model_state.partition_id, use_python_fusion_cache, - model_hash_str=model_hash_str, file_name=file_name, signature=signature, int_inputs=int_inputs), + model_hash_str=model_hash_str, file_name=file_name, int_inputs=int_inputs), ) shared.compiled_model_state.partition_id += 1 From 0e7977d84e2760319f597e50fe631fbcd586eee0 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 31 Dec 2024 10:41:56 -0500 Subject: [PATCH 200/249] fix lora on model change Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 1 + installer.py | 2 +- modules/model_flux.py | 8 +++++--- modules/sd_models.py | 5 +++++ modules/sd_samplers.py | 2 +- 5 files changed, 13 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0559b184a..46f993b1d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ NYE refresh release with quite a few optimizatios and bug fixes... - **Fixes**: - flux pipeline switches: txt/img/inpaint - flux custom unet loader for bnb + - flux do not requantize already quantized model - interrogate caption with T5 - on-the-fly quantization using TorchAO - remove concurrent preview requests diff --git a/installer.py b/installer.py index 60aab0ddd..de76ce17e 100644 --- a/installer.py +++ b/installer.py @@ -255,7 +255,7 @@ def uninstall(package, quiet = False): @lru_cache() -def pip(arg: str, ignore: bool = False, quiet: bool = False, uv = True): +def pip(arg: str, ignore: bool = False, quiet: bool = True, uv = True): originalArg = arg arg = arg.replace('>=', '==') package = arg.replace("install", "").replace("--upgrade", "").replace("--no-deps", "").replace("--force", "").replace(" ", " ").strip() diff --git a/modules/model_flux.py b/modules/model_flux.py index b89714d91..9b42de3e7 100644 --- a/modules/model_flux.py +++ b/modules/model_flux.py @@ -143,7 +143,9 @@ def quant_flux_bnb(checkpoint_info, transformer, text_encoder_2): """ -def load_quants(kwargs, repo_id, cache_dir): +def load_quants(kwargs, repo_id, cache_dir, allow_quant): + if not allow_quant: + return kwargs quant_args = {} quant_args = model_quant.create_bnb_config(quant_args) if quant_args: @@ -359,10 +361,10 @@ def load_flux(checkpoint_info, diffusers_load_config): # triggered by opts.sd_ch except Exception: pass - allow_quant = 'gguf' not in (sd_unet.loaded_unet or '') + allow_quant = 'gguf' not in (sd_unet.loaded_unet or '') and (quant is None or quant == 'none') fn = checkpoint_info.path if (fn is None) or (not os.path.exists(fn) or os.path.isdir(fn)): - kwargs = load_quants(kwargs, repo_id, cache_dir=shared.opts.diffusers_dir) + kwargs = load_quants(kwargs, repo_id, cache_dir=shared.opts.diffusers_dir, allow_quant=allow_quant) kwargs = model_quant.create_bnb_config(kwargs, allow_quant) kwargs = model_quant.create_ao_config(kwargs, allow_quant) if fn.endswith('.safetensors') and os.path.isfile(fn): diff --git a/modules/sd_models.py b/modules/sd_models.py index f962eeaca..e00913092 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -1622,6 +1622,11 @@ def unload_model_weights(op='model'): model_data.sd_model = None devices.torch_gc(force=True) shared.log.debug(f'Unload weights {op}: {memory_stats()}') + if not shared.opts.lora_legacy: + from modules.lora import networks + networks.loaded_networks.clear() + networks.previously_loaded_networks.clear() + networks.lora_cache.clear() elif op == 'refiner': if model_data.sd_refiner: if not shared.native: diff --git a/modules/sd_samplers.py b/modules/sd_samplers.py index 89c4e5edf..398f608d1 100644 --- a/modules/sd_samplers.py +++ b/modules/sd_samplers.py @@ -61,7 +61,7 @@ def create_sampler(name, model): model.prior_pipe.scheduler = copy.deepcopy(model.default_scheduler) model.prior_pipe.scheduler.config.clip_sample = False config = {k: v for k, v in model.scheduler.config.items() if not k.startswith('_')} - shared.log.debug(f'Sampler: default class={current}: {config}') + shared.log.debug(f'Sampler: "default" class={current}: {config}') if "flow" in model.scheduler.__class__.__name__.lower(): shared.state.prediction_type = "flow_prediction" elif hasattr(model.scheduler, "config") and hasattr(model.scheduler.config, "prediction_type"): From 78c9329099ce7d5bbd0eaee2232bdc0fed81eb13 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Tue, 31 Dec 2024 19:20:30 +0300 Subject: [PATCH 201/249] OpenVINO use model hash for signature --- modules/intel/openvino/__init__.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/modules/intel/openvino/__init__.py b/modules/intel/openvino/__init__.py index 88d579a94..1c89ceb48 100644 --- a/modules/intel/openvino/__init__.py +++ b/modules/intel/openvino/__init__.py @@ -360,7 +360,13 @@ def openvino_execute_partitioned(gm: GraphModule, *args, executor_parameters=Non ) model_hash_str = executor_parameters.get("model_hash_str", None) - signature = "signature" #str(id(gm)) + if file_name: + signature = file_name.rsplit("/", maxsplit=1)[-1].split("_fs", maxsplit=1)[0] + else: + signature = "signature" + if model_hash_str is None: + file_name = None + idx_minus = 0 int_inputs = [] for idx, input_data in enumerate(args): @@ -455,19 +461,19 @@ def openvino_fx(subgraph, example_inputs): dont_use_nncf = bool("Text Encoder" not in shared.opts.nncf_compress_weights) dont_use_quant = bool("Text Encoder" not in shared.opts.nncf_quantize) + # Create a hash to be used for caching + shared.compiled_model_state.model_hash_str = "" + subgraph.apply(generate_subgraph_str) + #shared.compiled_model_state.model_hash_str = shared.compiled_model_state.model_hash_str + sha256(subgraph.code.encode('utf-8')).hexdigest() + shared.compiled_model_state.model_hash_str = sha256(shared.compiled_model_state.model_hash_str.encode('utf-8')).hexdigest() + + # Check if the model was fully supported and already cached + example_inputs.reverse() + inputs_reversed = True + maybe_fs_cached_name = cached_model_name(shared.compiled_model_state.model_hash_str + "_fs", get_device(), example_inputs, shared.opts.openvino_cache_path) if not shared.opts.openvino_disable_model_caching: os.environ.setdefault('OPENVINO_TORCH_MODEL_CACHING', "1") - - # Create a hash to be used for caching - subgraph.apply(generate_subgraph_str) - shared.compiled_model_state.model_hash_str = shared.compiled_model_state.model_hash_str + sha256(subgraph.code.encode('utf-8')).hexdigest() - shared.compiled_model_state.model_hash_str = sha256(shared.compiled_model_state.model_hash_str.encode('utf-8')).hexdigest() - executor_parameters = {"model_hash_str": shared.compiled_model_state.model_hash_str} - # Check if the model was fully supported and already cached - example_inputs.reverse() - inputs_reversed = True - maybe_fs_cached_name = cached_model_name(shared.compiled_model_state.model_hash_str + "_fs", get_device(), example_inputs, shared.opts.openvino_cache_path) if os.path.isfile(maybe_fs_cached_name + ".xml") and os.path.isfile(maybe_fs_cached_name + ".bin"): example_inputs_reordered = [] @@ -510,7 +516,6 @@ def _call(*args): return _call else: os.environ.setdefault('OPENVINO_TORCH_MODEL_CACHING', "0") - maybe_fs_cached_name = None if inputs_reversed: example_inputs.reverse() From 42d932735bf550e1e690c4d4ba888a5e4f2de7e4 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 31 Dec 2024 11:41:25 -0500 Subject: [PATCH 202/249] ui networks initial sort Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 4 ++++ javascript/extraNetworks.js | 17 +++++++++++------ wiki | 2 +- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 46f993b1d..2730719a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,9 @@ NYE refresh release with quite a few optimizatios and bug fixes... - **OpenVINO**: - disable re-compile on resolution change - fix shape mismatch on resolution change +- **LoRA**: + - LoRA load/apply/unapply methods have been changed in 12/2024 Xmass release and further tuned in this release + - for details on available methods, see - **Fixes**: - flux pipeline switches: txt/img/inpaint - flux custom unet loader for bnb @@ -51,6 +54,7 @@ NYE refresh release with quite a few optimizatios and bug fixes... - corrections with batch processing - hires with refiner prompt and batch processing - processing with nested calls + - ui networks initial sort ## Update for 2024-12-24 diff --git a/javascript/extraNetworks.js b/javascript/extraNetworks.js index c42ba5761..3f27cbba2 100644 --- a/javascript/extraNetworks.js +++ b/javascript/extraNetworks.js @@ -1,5 +1,6 @@ const activePromptTextarea = {}; let sortVal = -1; +let totalCards = -1; // helpers @@ -226,8 +227,8 @@ function sortExtraNetworks(fixed = 'no') { if (fixed !== 'fixed') sortVal = (sortVal + 1) % sortDesc.length; for (const pg of pages) { const cards = Array.from(pg.querySelectorAll('.card') || []); - num = cards.length; - if (num === 0) return 'sort: no cards'; + if (cards.length === 0) return 'sort: no cards'; + num += cards.length; cards.sort((a, b) => { // eslint-disable-line no-loop-func switch (sortVal) { case 0: return 0; @@ -243,12 +244,12 @@ function sortExtraNetworks(fixed = 'no') { for (const card of cards) pg.appendChild(card); } const desc = sortDesc[sortVal]; - log('sortExtraNetworks', { name: pagename, val: sortVal, order: desc, fixed: fixed === 'fixed', items: num }); + log('sortNetworks', { name: pagename, val: sortVal, order: desc, fixed: fixed === 'fixed', items: num }); return desc; } function refreshENInput(tabname) { - log('refreshExtraNetworks', tabname, gradioApp().querySelector(`#${tabname}_extra_networks textarea`)?.value); + log('refreshNetworks', tabname, gradioApp().querySelector(`#${tabname}_extra_networks textarea`)?.value); gradioApp().querySelector(`#${tabname}_extra_networks textarea`)?.dispatchEvent(new Event('input')); } @@ -467,11 +468,15 @@ function setupExtraNetworksForTab(tabname) { el.parentElement.style.width = '-webkit-fill-available'; } } + const cards = Array.from(gradioApp().querySelectorAll('.extra-network-cards > .card')); + if (cards.length > 0 && cards.length !== totalCards) { + totalCards = cards.length; + sortExtraNetworks('fixed'); + } if (lastView !== entries[0].intersectionRatio > 0) { lastView = entries[0].intersectionRatio > 0; if (lastView) { refreshENpage(); - // sortExtraNetworks('fixed'); if (window.opts.extra_networks_card_cover === 'cover') { en.style.position = 'absolute'; en.style.height = 'unset'; @@ -536,5 +541,5 @@ async function setupExtraNetworks() { registerPrompt('img2img', 'img2img_neg_prompt'); registerPrompt('control', 'control_prompt'); registerPrompt('control', 'control_neg_prompt'); - log('initExtraNetworks'); + log('initNetworks'); } diff --git a/wiki b/wiki index 7d159b39f..9eaaec772 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 7d159b39f70b1e6cc5673b39180b50e6688450f3 +Subproject commit 9eaaec7728c4b23a6efe78fa643cd5655ce8801f From 296ce0284deb28c690865b738e5c88695bb1f7af Mon Sep 17 00:00:00 2001 From: Disty0 Date: Tue, 31 Dec 2024 19:41:50 +0300 Subject: [PATCH 203/249] Fix ESRGAN with CPU --- CHANGELOG.md | 1 + modules/postprocess/esrgan_model.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2730719a5..5fc767179 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,6 +55,7 @@ NYE refresh release with quite a few optimizatios and bug fixes... - hires with refiner prompt and batch processing - processing with nested calls - ui networks initial sort + - esrgan on cpu devices ## Update for 2024-12-24 diff --git a/modules/postprocess/esrgan_model.py b/modules/postprocess/esrgan_model.py index 8ff75abab..41a3585bd 100644 --- a/modules/postprocess/esrgan_model.py +++ b/modules/postprocess/esrgan_model.py @@ -142,7 +142,7 @@ def load_model(self, path: str): if self.models.get(info.local_data_path, None) is not None: shared.log.debug(f"Upscaler cached: type={self.name} model={info.local_data_path}") return self.models[info.local_data_path] - state_dict = torch.load(info.local_data_path, map_location='cpu' if devices.device.type == 'mps' else None) + state_dict = torch.load(info.local_data_path, map_location='cpu' if devices.device.type in {'mps', 'cpu'} else None) shared.log.info(f"Upscaler loaded: type={self.name} model={info.local_data_path}") if "params_ema" in state_dict: From 12b81adcd5ebe60c60202b09f939e6858d11dea1 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Tue, 31 Dec 2024 19:56:03 +0300 Subject: [PATCH 204/249] Don't reset dynamo on upcaler compile --- modules/upscaler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/upscaler.py b/modules/upscaler.py index 744e60052..bda39f858 100644 --- a/modules/upscaler.py +++ b/modules/upscaler.py @@ -217,7 +217,6 @@ def compile_upscaler(model): try: if "Upscaler" in shared.opts.cuda_compile and shared.opts.cuda_compile_backend != 'none': import torch._dynamo # pylint: disable=unused-import,redefined-outer-name - torch._dynamo.reset() # pylint: disable=protected-access if shared.opts.cuda_compile_backend not in torch._dynamo.list_backends(): # pylint: disable=protected-access shared.log.warning(f"Upscaler compile not available: backend={shared.opts.cuda_compile_backend} available={torch._dynamo.list_backends()}") # pylint: disable=protected-access return model From 3808c620c476cadc9da84365d95d6e557dae6dae Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 31 Dec 2024 12:08:18 -0500 Subject: [PATCH 205/249] preview use per-job lock instead of global Signed-off-by: Vladimir Mandic --- modules/progress.py | 2 +- modules/shared_state.py | 18 +++++++----------- wiki | 2 +- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/modules/progress.py b/modules/progress.py index 8e01c02b3..b90981c3c 100644 --- a/modules/progress.py +++ b/modules/progress.py @@ -78,7 +78,7 @@ def progressapi(req: ProgressRequest): id_live_preview = req.id_live_preview live_preview = None updated = shared.state.set_current_image() - debug_log(f'Preview: job={shared.state.job} active={active} progress={current}/{total} step={shared.state.current_image_sampling_step}/{shared.state.sampling_step} request={id_live_preview} last={shared.state.id_live_preview} enabled={shared.opts.live_previews_enable} busy={shared.state.preview_busy} updated={updated} image={shared.state.current_image} elapsed={elapsed:.3f}') + debug_log(f'Preview: job={shared.state.job} active={active} progress={current}/{total} step={shared.state.current_image_sampling_step}/{shared.state.sampling_step} request={id_live_preview} last={shared.state.id_live_preview} enabled={shared.opts.live_previews_enable} job={shared.state.preview_job} updated={updated} image={shared.state.current_image} elapsed={elapsed:.3f}') if not active: return InternalProgressResponse(job=shared.state.job, active=active, queued=queued, paused=paused, completed=completed, id_live_preview=-1, debug=debug, textinfo="Queued..." if queued else "Waiting...") if shared.opts.live_previews_enable and (shared.state.id_live_preview != id_live_preview) and (shared.state.current_image is not None): diff --git a/modules/shared_state.py b/modules/shared_state.py index dd3905de9..9f56b14e0 100644 --- a/modules/shared_state.py +++ b/modules/shared_state.py @@ -27,7 +27,7 @@ class State: prediction_type = "epsilon" api = False disable_preview = False - preview_busy = False + preview_job = -1 time_start = None need_restart = False server_start = time.time() @@ -113,7 +113,7 @@ def begin(self, title="", api=None): self.current_sigma_next = None self.id_live_preview = 0 self.interrupted = False - self.preview_busy = False + self.preview_job = -1 self.job = title self.job_count = -1 self.frame_count = -1 @@ -136,15 +136,11 @@ def end(self, api=None): # fn = f'{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access # log.debug(f'Access state.end: {fn}') # pylint: disable=protected-access self.time_start = time.time() - if self.debug_output: - if self.preview_busy: - log.debug('State end: preview busy') - log.debug(f'State end: {self.job} time={time.time() - self.time_start:.2f}') self.job = "" self.job_count = 0 self.job_no = 0 self.frame_count = 0 - self.preview_busy = False + self.preview_job = -1 self.paused = False self.interrupted = False self.skipped = False @@ -162,10 +158,10 @@ def set_current_image(self): return False def do_set_current_image(self): - if (self.current_latent is None) or self.disable_preview or self.preview_busy: + if (self.current_latent is None) or self.disable_preview or (self.preview_job == self.job_no): return False from modules import shared, sd_samplers - self.preview_busy = True + self.preview_job = self.job_no try: sample = self.current_latent self.current_image_sampling_step = self.sampling_step @@ -180,10 +176,10 @@ def do_set_current_image(self): pass # ignore sigma errors image = sd_samplers.samples_to_image_grid(sample) if shared.opts.show_progress_grid else sd_samplers.sample_to_image(sample) self.assign_current_image(image) - self.preview_busy = False + self.preview_job = -1 return True except Exception as e: - self.preview_busy = False + self.preview_job = -1 log.error(f'State image: last={self.id_live_preview} step={self.sampling_step} {e}') display(e, 'State image') return False diff --git a/wiki b/wiki index 9eaaec772..9f132c033 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 9eaaec7728c4b23a6efe78fa643cd5655ce8801f +Subproject commit 9f132c033e8ed20c34bb3919b0da1c65c2a9c30a From b0f1848fad8251aaa1c8178adbc669532e6a86dc Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 31 Dec 2024 12:27:20 -0500 Subject: [PATCH 206/249] upadate wiki Signed-off-by: Vladimir Mandic --- extensions-builtin/sdnext-modernui | 2 +- wiki | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/extensions-builtin/sdnext-modernui b/extensions-builtin/sdnext-modernui index c3d8d9170..62d8a54a7 160000 --- a/extensions-builtin/sdnext-modernui +++ b/extensions-builtin/sdnext-modernui @@ -1 +1 @@ -Subproject commit c3d8d9170e16e029e31ef20d065fcfb77f9c64fc +Subproject commit 62d8a54a7ec24a6fbf69697da18e67035754d072 diff --git a/wiki b/wiki index 9f132c033..fc38907c8 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 9f132c033e8ed20c34bb3919b0da1c65c2a9c30a +Subproject commit fc38907c8336b7d172cca2ba056b3b55078fc0df From 5abcb6eb688661f20c8333ffc5dbbfd2d955797a Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 31 Dec 2024 12:28:53 -0500 Subject: [PATCH 207/249] lint fixes Signed-off-by: Vladimir Mandic --- modules/sd_models_compile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/sd_models_compile.py b/modules/sd_models_compile.py index ebc0460fe..0972c3350 100644 --- a/modules/sd_models_compile.py +++ b/modules/sd_models_compile.py @@ -531,7 +531,7 @@ def torchao_model(model, op=None, sd_model=None): # pylint: disable=unused-argum return sd_model -def openvino_recompile_model(p, hires=False, refiner=False): # recompile if a parameter changes +def openvino_recompile_model(p, hires=False, refiner=False): # recompile if a parameter changes # pylint: disable=unused-argument if shared.opts.cuda_compile_backend == "openvino_fx" and 'Model' in shared.opts.cuda_compile: compile_height = p.height if not hires and hasattr(p, 'height') else p.hr_upscale_to_y compile_width = p.width if not hires and hasattr(p, 'width') else p.hr_upscale_to_x From daee93e50d2dd64602327437ad9d44cf764606f4 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 31 Dec 2024 12:38:08 -0500 Subject: [PATCH 208/249] add commit info to changelog Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5fc767179..b383a90c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,12 +2,15 @@ ## Update for 2024-12-31 -NYE refresh release with quite a few optimizatios and bug fixes... +NYE refresh release with quite a few optimizatios and bug fixes... +Commit hash: `master: #dcfc9f3` `dev: #935cac6` - **LoRA**: + - LoRA load/apply/unapply methods have been changed in 12/2024 Xmass release and further tuned in this release + - for details on available methods, see - **Sana** support - quantized models support - - fuse support with on-demand apply/unapply + - add fuse support with on-demand apply/unapply (new default) - add legacy option in *settings -> networks* - **HunyuanVideo**: - optimizations: full offload, quantization and tiling support @@ -33,9 +36,6 @@ NYE refresh release with quite a few optimizatios and bug fixes... - **OpenVINO**: - disable re-compile on resolution change - fix shape mismatch on resolution change -- **LoRA**: - - LoRA load/apply/unapply methods have been changed in 12/2024 Xmass release and further tuned in this release - - for details on available methods, see - **Fixes**: - flux pipeline switches: txt/img/inpaint - flux custom unet loader for bnb From 39723c32fb5ed4f8a2179d425654def2eebfcd5d Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 31 Dec 2024 15:52:00 -0500 Subject: [PATCH 209/249] update lora changed Signed-off-by: Vladimir Mandic --- modules/lora/extra_networks_lora.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/lora/extra_networks_lora.py b/modules/lora/extra_networks_lora.py index d9b8f81f2..e89e57b6e 100644 --- a/modules/lora/extra_networks_lora.py +++ b/modules/lora/extra_networks_lora.py @@ -130,7 +130,7 @@ def changed(self, requested: List[str], include: List[str], exclude: List[str]): key = f'{",".join(include)}:{",".join(exclude)}' loaded = sd_model.loaded_loras.get(key, []) # shared.log.trace(f'Load network: type=LoRA key="{key}" requested={requested} loaded={loaded}') - if len(requested) != len(loaded): + if (len(requested) == 0) or (len(requested) != len(loaded)): sd_model.loaded_loras[key] = requested return True for r, l in zip(requested, loaded): From e1b190ed8a4fbf6b2edcec30153ab9b5fd9c9564 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 1 Jan 2025 09:49:08 -0500 Subject: [PATCH 210/249] first update in 2025 Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 15 +++++++++++++++ installer.py | 13 +++++++------ modules/cmd_args.py | 2 -- modules/postprocess/yolo.py | 30 +++++++++++++++++------------- modules/progress.py | 2 ++ modules/sd_models.py | 17 ++++++++++++----- modules/shared.py | 1 + modules/timer.py | 20 +++++++++++++++----- modules/ui_control.py | 2 +- webui.py | 2 +- wiki | 2 +- 11 files changed, 72 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b383a90c0..a8040f096 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ # Change Log for SD.Next +## Update for 2025-01-01 + +- **Logging**: + - reverted enable debug by default + - updated [debug wiki](https://github.com/vladmandic/automatic/wiki/debug) + - sort logged timers by duration + - allow min duration env variable for timers: `SD_MIN_TIMER=0.1` (default) + - update installer messages +- **Detailer**: + - add explicit detailer steps setting +- **Fixes**: + - explict clear caches on model load + - lock adetailer commit: `#a89c01d` + - xyzgrid fix progress calculation + ## Update for 2024-12-31 NYE refresh release with quite a few optimizatios and bug fixes... diff --git a/installer.py b/installer.py index de76ce17e..11e7f1d94 100644 --- a/installer.py +++ b/installer.py @@ -54,7 +54,7 @@ class Dot(dict): # dot notation access to dictionary attributes diffusers_commit = "unknown" extensions_commit = { 'sd-webui-controlnet': 'ecd33eb', - # 'adetailer': 'a89c01d' + 'adetailer': 'a89c01d' # 'stable-diffusion-webui-images-browser': '27fe4a7', } @@ -230,16 +230,16 @@ def installed(package, friendly: str = None, reload = False, quiet = False): exact = pkg_version == p[1] if not exact and not quiet: if args.experimental: - log.warning(f"Package: {p[0]} installed={pkg_version} required={p[1]} allowing experimental") + log.warning(f'Install: package="{p[0]}" installed={pkg_version} required={p[1]} allowing experimental') else: - log.warning(f"Package: {p[0]} installed={pkg_version} required={p[1]} version mismatch") + log.warning(f'Install: package="{p[0]}" installed={pkg_version} required={p[1]} version mismatch') ok = ok and (exact or args.experimental) else: if not quiet: - log.debug(f"Package: {p[0]} not found") + log.debug(f'Install: package="{p[0]}" install required') return ok except Exception as e: - log.error(f"Package: {pkgs} {e}") + log.error(f'Install: package="{pkgs}" {e}') return False @@ -1342,9 +1342,10 @@ def add_args(parser): group_log = parser.add_argument_group('Logging') group_log.add_argument("--log", type=str, default=os.environ.get("SD_LOG", None), help="Set log file, default: %(default)s") - # group_log.add_argument('--debug', default=os.environ.get("SD_DEBUG",False), action='store_true', help="Run installer with debug logging, default: %(default)s") + group_log.add_argument('--debug', default=os.environ.get("SD_DEBUG",False), action='store_true', help="Run installer with debug logging, default: %(default)s") group_log.add_argument("--profile", default=os.environ.get("SD_PROFILE", False), action='store_true', help="Run profiler, default: %(default)s") group_log.add_argument('--docs', default=os.environ.get("SD_DOCS", False), action='store_true', help="Mount API docs, default: %(default)s") + group_log.add_argument("--api-log", default=os.environ.get("SD_APILOG", True), action='store_true', help="Log all API requests") def parse_args(parser): diff --git a/modules/cmd_args.py b/modules/cmd_args.py index 9fb5a3fb7..018d802ee 100644 --- a/modules/cmd_args.py +++ b/modules/cmd_args.py @@ -74,9 +74,7 @@ def compatibility_args(): group_compat.add_argument("--disable-extension-access", default=False, action='store_true', help=argparse.SUPPRESS) group_compat.add_argument("--api", action='store_true', help=argparse.SUPPRESS, default=True) group_compat.add_argument("--api-auth", type=str, help=argparse.SUPPRESS, default=None) - group_compat.add_argument("--api-log", default=os.environ.get("SD_APILOG", True), action='store_true', help=argparse.SUPPRESS) group_compat.add_argument("--disable-queue", default=os.environ.get("SD_DISABLEQUEUE", False), action='store_true', help=argparse.SUPPRESS) - group_compat.add_argument('--debug', default=os.environ.get("SD_DEBUG", True), action='store_true', help=argparse.SUPPRESS) diff --git a/modules/postprocess/yolo.py b/modules/postprocess/yolo.py index caa711651..3ee2406d1 100644 --- a/modules/postprocess/yolo.py +++ b/modules/postprocess/yolo.py @@ -231,7 +231,7 @@ def restore(self, np_image, p: processing.StableDiffusionProcessing = None): 'negative_prompt': negative, 'denoising_strength': shared.opts.detailer_strength, 'sampler_name': orig_p.get('hr_sampler_name', 'default'), - 'steps': orig_p.get('refiner_steps', 0), + 'steps': shared.opts.detailer_steps, 'styles': [], 'inpaint_full_res': True, 'inpainting_mask_invert': 0, @@ -308,7 +308,7 @@ def restore(self, np_image, p: processing.StableDiffusionProcessing = None): return np_image def ui(self, tab: str): - def ui_settings_change(detailers, classes, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou): + def ui_settings_change(detailers, classes, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou, steps): shared.opts.detailer_models = detailers shared.opts.detailer_classes = classes shared.opts.detailer_strength = strength @@ -319,8 +319,9 @@ def ui_settings_change(detailers, classes, strength, padding, blur, min_confiden shared.opts.detailer_min_size = min_size shared.opts.detailer_max_size = max_size shared.opts.detailer_iou = iou + shared.opts.detailer_steps = steps shared.opts.save(shared.config_filename, silent=True) - shared.log.debug(f'Detailer settings: models={shared.opts.detailer_models} classes={shared.opts.detailer_classes} strength={shared.opts.detailer_strength} conf={shared.opts.detailer_conf} max={shared.opts.detailer_max} iou={shared.opts.detailer_iou} size={shared.opts.detailer_min_size}-{shared.opts.detailer_max_size} padding={shared.opts.detailer_padding}') + shared.log.debug(f'Detailer settings: models={shared.opts.detailer_models} classes={shared.opts.detailer_classes} strength={shared.opts.detailer_strength} conf={shared.opts.detailer_conf} max={shared.opts.detailer_max} iou={shared.opts.detailer_iou} size={shared.opts.detailer_min_size}-{shared.opts.detailer_max_size} padding={shared.opts.detailer_padding} steps={shared.opts.detailer_steps}') with gr.Accordion(open=False, label="Detailer", elem_id=f"{tab}_detailer_accordion", elem_classes=["small-accordion"], visible=shared.native): with gr.Row(): @@ -331,7 +332,9 @@ def ui_settings_change(detailers, classes, strength, padding, blur, min_confiden with gr.Row(): classes = gr.Textbox(label="Classes", placeholder="Classes", elem_id=f"{tab}_detailer_classes") with gr.Row(): + steps = gr.Slider(label="Detailer steps", elem_id=f"{tab}_detailer_steps", value=shared.opts.detailer_steps, min=0, max=99, step=1) strength = gr.Slider(label="Detailer strength", elem_id=f"{tab}_detailer_strength", value=shared.opts.detailer_strength, minimum=0, maximum=1, step=0.01) + with gr.Row(): max_detected = gr.Slider(label="Max detected", elem_id=f"{tab}_detailer_max", value=shared.opts.detailer_max, min=1, maximum=10, step=1) with gr.Row(): padding = gr.Slider(label="Edge padding", elem_id=f"{tab}_detailer_padding", value=shared.opts.detailer_padding, minimum=0, maximum=100, step=1) @@ -344,16 +347,17 @@ def ui_settings_change(detailers, classes, strength, padding, blur, min_confiden min_size = gr.Slider(label="Min size", elem_id=f"{tab}_detailer_min_size", value=min_size, minimum=0.0, maximum=1.0, step=0.05) max_size = shared.opts.detailer_max_size if shared.opts.detailer_max_size < 1 and shared.opts.detailer_max_size > 0 else 1.0 max_size = gr.Slider(label="Max size", elem_id=f"{tab}_detailer_max_size", value=max_size, minimum=0.0, maximum=1.0, step=0.05) - detailers.change(fn=ui_settings_change, inputs=[detailers, classes, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou], outputs=[]) - classes.change(fn=ui_settings_change, inputs=[detailers, classes, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou], outputs=[]) - strength.change(fn=ui_settings_change, inputs=[detailers, classes, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou], outputs=[]) - padding.change(fn=ui_settings_change, inputs=[detailers, classes, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou], outputs=[]) - blur.change(fn=ui_settings_change, inputs=[detailers, classes, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou], outputs=[]) - min_confidence.change(fn=ui_settings_change, inputs=[detailers, classes, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou], outputs=[]) - max_detected.change(fn=ui_settings_change, inputs=[detailers, classes, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou], outputs=[]) - min_size.change(fn=ui_settings_change, inputs=[detailers, classes, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou], outputs=[]) - max_size.change(fn=ui_settings_change, inputs=[detailers, classes, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou], outputs=[]) - iou.change(fn=ui_settings_change, inputs=[detailers, classes, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou], outputs=[]) + detailers.change(fn=ui_settings_change, inputs=[detailers, classes, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou, steps], outputs=[]) + classes.change(fn=ui_settings_change, inputs=[detailers, classes, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou, steps], outputs=[]) + strength.change(fn=ui_settings_change, inputs=[detailers, classes, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou, steps], outputs=[]) + padding.change(fn=ui_settings_change, inputs=[detailers, classes, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou, steps], outputs=[]) + blur.change(fn=ui_settings_change, inputs=[detailers, classes, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou, steps], outputs=[]) + min_confidence.change(fn=ui_settings_change, inputs=[detailers, classes, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou, steps], outputs=[]) + max_detected.change(fn=ui_settings_change, inputs=[detailers, classes, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou, steps], outputs=[]) + min_size.change(fn=ui_settings_change, inputs=[detailers, classes, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou, steps], outputs=[]) + max_size.change(fn=ui_settings_change, inputs=[detailers, classes, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou, steps], outputs=[]) + iou.change(fn=ui_settings_change, inputs=[detailers, classes, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou, steps], outputs=[]) + steps.change(fn=ui_settings_change, inputs=[detailers, classes, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou, steps], outputs=[]) return enabled diff --git a/modules/progress.py b/modules/progress.py index b90981c3c..bc3e5500c 100644 --- a/modules/progress.py +++ b/modules/progress.py @@ -71,6 +71,8 @@ def progressapi(req: ProgressRequest): step_y = max(shared.state.sampling_steps, 1) current = step_y * batch_x + step_x total = step_y * batch_y + while total < current: + total += step_y progress = min(1, abs(current / total) if total > 0 else 0) elapsed = time.time() - shared.state.time_start if shared.state.time_start is not None else 0 predicted = elapsed / progress if progress > 0 else None diff --git a/modules/sd_models.py b/modules/sd_models.py index e00913092..46463a85a 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -270,6 +270,7 @@ def set_diffuser_options(sd_model, vae = None, op: str = 'model', offload=True): if not (hasattr(sd_model, "has_accelerate") and sd_model.has_accelerate): sd_model.has_accelerate = False + clear_caches() set_vae_options(sd_model, vae, op) set_diffusers_attention(sd_model) @@ -1605,6 +1606,17 @@ def disable_offload(sd_model): sd_model.has_accelerate = False +def clear_caches(): + shared.log.debug('Cache clear') + if not shared.opts.lora_legacy: + from modules.lora import networks + networks.loaded_networks.clear() + networks.previously_loaded_networks.clear() + networks.lora_cache.clear() + from modules import prompt_parser_diffusers + prompt_parser_diffusers.cache.clear() + + def unload_model_weights(op='model'): if shared.compiled_model_state is not None: shared.compiled_model_state.compiled_cache.clear() @@ -1622,11 +1634,6 @@ def unload_model_weights(op='model'): model_data.sd_model = None devices.torch_gc(force=True) shared.log.debug(f'Unload weights {op}: {memory_stats()}') - if not shared.opts.lora_legacy: - from modules.lora import networks - networks.loaded_networks.clear() - networks.previously_loaded_networks.clear() - networks.lora_cache.clear() elif op == 'refiner': if model_data.sd_refiner: if not shared.native: diff --git a/modules/shared.py b/modules/shared.py index 6aa5d9743..a02c01abf 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -861,6 +861,7 @@ def get_default_modes(): "detailer_max_size": OptionInfo(1.0, "Max object size", gr.Slider, {"minimum": 0.1, "maximum": 1, "step": 0.05, "visible": False}), "detailer_padding": OptionInfo(20, "Item padding", gr.Slider, {"minimum": 0, "maximum": 100, "step": 1, "visible": False}), "detailer_blur": OptionInfo(10, "Item edge blur", gr.Slider, {"minimum": 0, "maximum": 100, "step": 1, "visible": False}), + "detailer_steps": OptionInfo(10, "Detailer steps", gr.Slider, {"minimum": 0, "maximum": 99, "step": 1, "visible": False}), "detailer_strength": OptionInfo(0.5, "Detailer strength", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01, "visible": False}), "detailer_models": OptionInfo(['face-yolo8n'], "Detailer models", gr.Dropdown, lambda: {"multiselect":True, "choices": list(yolo.list), "visible": False}), "detailer_unload": OptionInfo(False, "Move detailer model to CPU when complete"), diff --git a/modules/timer.py b/modules/timer.py index 43e859140..977206e06 100644 --- a/modules/timer.py +++ b/modules/timer.py @@ -1,7 +1,14 @@ +import os import time import sys +try: + default_min_time = float(os.environ.get('SD_MIN_TIMER', '0.05')) +except Exception: + default_min_time = 0.1 + + class Timer: def __init__(self): self.start = time.time() @@ -31,20 +38,23 @@ def record(self, category=None, extra_time=0, reset=True): self.records[category] += e + extra_time self.total += e + extra_time - def summary(self, min_time=0.05, total=True): + def summary(self, min_time=default_min_time, total=True): if self.profile: min_time = -1 - res = f"{self.total:.2f} " if total else '' + res = f"total={self.total:.2f} " if total else '' additions = [x for x in self.records.items() if x[1] >= min_time] + additions = sorted(additions, key=lambda x: x[1], reverse=True) if not additions: return res res += " ".join([f"{category}={time_taken:.2f}" for category, time_taken in additions]) return res - def dct(self, min_time=0.05): + def dct(self, min_time=default_min_time): if self.profile: - return {k: round(v, 4) for k, v in self.records.items()} - return {k: round(v, 2) for k, v in self.records.items() if v >= min_time} + res = {k: round(v, 4) for k, v in self.records.items()} + res = {k: round(v, 2) for k, v in self.records.items() if v >= min_time} + res = {k: v for k, v in sorted(res.items(), key=lambda x: x[1], reverse=True)} # noqa: C416 + return res def reset(self): self.__init__() diff --git a/modules/ui_control.py b/modules/ui_control.py index cd78313a7..c12a03130 100644 --- a/modules/ui_control.py +++ b/modules/ui_control.py @@ -29,7 +29,7 @@ def return_stats(t: float = None): elapsed_m = int(elapsed // 60) elapsed_s = elapsed % 60 elapsed_text = f"Time: {elapsed_m}m {elapsed_s:.2f}s |" if elapsed_m > 0 else f"Time: {elapsed_s:.2f}s |" - summary = timer.process.summary(min_time=0.25, total=False).replace('=', ' ') + summary = timer.process.summary(total=False).replace('=', ' ') gpu = '' cpu = '' if not shared.mem_mon.disabled: diff --git a/webui.py b/webui.py index 9c85fe426..3c8361ecf 100644 --- a/webui.py +++ b/webui.py @@ -244,7 +244,7 @@ def start_common(): def start_ui(): - log.info('UI start') + log.debug('UI start sequence') modules.script_callbacks.before_ui_callback() timer.startup.record("before-ui") shared.demo = modules.ui.create_ui(timer.startup) diff --git a/wiki b/wiki index fc38907c8..7bd8f8200 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit fc38907c8336b7d172cca2ba056b3b55078fc0df +Subproject commit 7bd8f82008007bc7a766ca47b4dc1a54470397df From 35e41c26065ee1fc2444f5497b0a8c1e0a2d5efb Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 1 Jan 2025 12:29:31 -0500 Subject: [PATCH 211/249] allegro Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 5 ++ README.md | 8 +- modules/processing_diffusers.py | 2 - scripts/allegrovideo.py | 151 ++++++++++++++++++++++++++++++++ 4 files changed, 157 insertions(+), 9 deletions(-) create mode 100644 scripts/allegrovideo.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a8040f096..2f49388a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ ## Update for 2025-01-01 +- [Allegro Video](https://huggingface.co/rhymes-ai/Allegro) + - optimizations: full offload and quantization support + - *reference values*: width 1280 height 720 frames 88 steps 100 guidance 7.5 + - *note*: allegro model is really sensitive to input width/height/frames/steps + and may result in completely corrupt output if those are not within expected range - **Logging**: - reverted enable debug by default - updated [debug wiki](https://github.com/vladmandic/automatic/wiki/debug) diff --git a/README.md b/README.md index 2a95373d0..a612f6127 100644 --- a/README.md +++ b/README.md @@ -30,11 +30,9 @@ All individual features are not listed here, instead check [ChangeLog](CHANGELOG - Built-in Control for Text, Image, Batch and video processing! - Multiplatform! ▹ **Windows | Linux | MacOS | nVidia | AMD | IntelArc/IPEX | DirectML | OpenVINO | ONNX+Olive | ZLUDA** -- Multiple backends! - ▹ **Diffusers | Original** - Platform specific autodetection and tuning performed on install - Optimized processing with latest `torch` developments with built-in support for `torch.compile` - and multiple compile backends: *Triton, ZLUDA, StableFast, DeepCache, OpenVINO, NNCF, IPEX, OneDiff* + and multiple compile backends: *Triton, StableFast, DeepCache, NNCF, OneDiff* - Built-in queue management - Built in installer with automatic updates and dependency management - Mobile compatible @@ -83,10 +81,6 @@ SD.Next supports broad range of models: [supported models](https://vladmandic.gi > [!WARNING] > If you run into issues, check out [troubleshooting](https://vladmandic.github.io/sdnext-docs/Troubleshooting/) and [debugging](https://vladmandic.github.io/sdnext-docs/Debug/) guides -> [!TIP] -> All command line options can also be set via env variable -> For example `--debug` is same as `set SD_DEBUG=true` - ### Credits - Main credit goes to [Automatic1111 WebUI](https://github.com/AUTOMATIC1111/stable-diffusion-webui) for the original codebase diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index bd759e5f0..d978cbe2b 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -91,8 +91,6 @@ def process_base(p: processing.StableDiffusionProcessing): sd_models.move_model(shared.sd_model.transformer, devices.device) extra_networks.activate(p, exclude=['text_encoder', 'text_encoder_2', 'text_encoder_3']) hidiffusion.apply(p, shared.sd_model_type) - # if 'image' in base_args: - # base_args['image'] = set_latents(p) timer.process.record('move') if hasattr(shared.sd_model, 'tgate') and getattr(p, 'gate_step', -1) > 0: base_args['gate_step'] = p.gate_step diff --git a/scripts/allegrovideo.py b/scripts/allegrovideo.py new file mode 100644 index 000000000..675c6fb06 --- /dev/null +++ b/scripts/allegrovideo.py @@ -0,0 +1,151 @@ +import time +import gradio as gr +import transformers +import diffusers +from modules import scripts, processing, shared, images, devices, sd_models, sd_checkpoint, model_quant, timer + + +repo_id = 'rhymes-ai/Allegro' + + +def hijack_decode(*args, **kwargs): + t0 = time.time() + vae: diffusers.AutoencoderKLAllegro = shared.sd_model.vae + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model, exclude=['vae']) + res = shared.sd_model.vae.orig_decode(*args, **kwargs) + t1 = time.time() + timer.process.add('vae', t1-t0) + shared.log.debug(f'Video: vae={vae.__class__.__name__} time={t1-t0:.2f}') + return res + + +def hijack_encode_prompt(*args, **kwargs): + t0 = time.time() + res = shared.sd_model.vae.orig_encode_prompt(*args, **kwargs) + t1 = time.time() + timer.process.add('te', t1-t0) + shared.log.debug(f'Video: te={shared.sd_model.text_encoder.__class__.__name__} time={t1-t0:.2f}') + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) + return res + + +class Script(scripts.Script): + def title(self): + return 'Video: Allegro' + + def show(self, is_img2img): + return not is_img2img if shared.native else False + + # return signature is array of gradio components + def ui(self, _is_img2img): + def video_type_change(video_type): + return [ + gr.update(visible=video_type != 'None'), + gr.update(visible=video_type == 'GIF' or video_type == 'PNG'), + gr.update(visible=video_type == 'MP4'), + gr.update(visible=video_type == 'MP4'), + ] + + with gr.Row(): + gr.HTML('  Allegro Video
') + with gr.Row(): + num_frames = gr.Slider(label='Frames', minimum=4, maximum=88, step=1, value=22) + with gr.Row(): + override_scheduler = gr.Checkbox(label='Override scheduler', value=True) + with gr.Row(): + video_type = gr.Dropdown(label='Video file', choices=['None', 'GIF', 'PNG', 'MP4'], value='None') + duration = gr.Slider(label='Duration', minimum=0.25, maximum=10, step=0.25, value=2, visible=False) + with gr.Row(): + gif_loop = gr.Checkbox(label='Loop', value=True, visible=False) + mp4_pad = gr.Slider(label='Pad frames', minimum=0, maximum=24, step=1, value=1, visible=False) + mp4_interpolate = gr.Slider(label='Interpolate frames', minimum=0, maximum=24, step=1, value=0, visible=False) + video_type.change(fn=video_type_change, inputs=[video_type], outputs=[duration, gif_loop, mp4_pad, mp4_interpolate]) + return [num_frames, override_scheduler, video_type, duration, gif_loop, mp4_pad, mp4_interpolate] + + def run(self, p: processing.StableDiffusionProcessing, num_frames, override_scheduler, video_type, duration, gif_loop, mp4_pad, mp4_interpolate): # pylint: disable=arguments-differ, unused-argument + # set params + num_frames = int(num_frames) + p.width = 8 * int(p.width // 8) + p.height = 8 * int(p.height // 8) + p.do_not_save_grid = True + p.ops.append('video') + + # load model + if shared.sd_model.__class__ != diffusers.AllegroPipeline: + sd_models.unload_model_weights() + t0 = time.time() + quant_args = {} + quant_args = model_quant.create_bnb_config(quant_args) + if quant_args: + model_quant.load_bnb(f'Load model: type=Allegro quant={quant_args}') + if not quant_args: + quant_args = model_quant.create_ao_config(quant_args) + if quant_args: + model_quant.load_torchao(f'Load model: type=Allegro quant={quant_args}') + transformer = diffusers.AllegroTransformer3DModel.from_pretrained( + repo_id, + subfolder="transformer", + torch_dtype=devices.dtype, + cache_dir=shared.opts.hfcache_dir, + **quant_args + ) + shared.log.debug(f'Video: module={transformer.__class__.__name__}') + text_encoder = transformers.T5EncoderModel.from_pretrained( + repo_id, + subfolder="text_encoder", + cache_dir=shared.opts.hfcache_dir, + torch_dtype=devices.dtype, + **quant_args + ) + shared.log.debug(f'Video: module={text_encoder.__class__.__name__}') + shared.sd_model = diffusers.AllegroPipeline.from_pretrained( + repo_id, + # transformer=transformer, + # text_encoder=text_encoder, + cache_dir=shared.opts.hfcache_dir, + torch_dtype=devices.dtype, + **quant_args + ) + t1 = time.time() + shared.log.debug(f'Video: load cls={shared.sd_model.__class__.__name__} repo="{repo_id}" dtype={devices.dtype} time={t1-t0:.2f}') + sd_models.set_diffuser_options(shared.sd_model) + shared.sd_model.sd_checkpoint_info = sd_checkpoint.CheckpointInfo(repo_id) + shared.sd_model.sd_model_hash = None + shared.sd_model.vae.orig_decode = shared.sd_model.vae.decode + shared.sd_model.vae.orig_encode_prompt = shared.sd_model.encode_prompt + shared.sd_model.vae.decode = hijack_decode + shared.sd_model.encode_prompt = hijack_encode_prompt + shared.sd_model.vae.enable_tiling() + # shared.sd_model.vae.enable_slicing() + + shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) + devices.torch_gc(force=True) + + processing.fix_seed(p) + if override_scheduler: + p.sampler_name = 'Default' + p.steps = 100 + p.task_args['num_frames'] = num_frames + p.task_args['output_type'] = 'pil' + p.task_args['clean_caption'] = False + + p.all_prompts, p.all_negative_prompts = shared.prompt_styles.apply_styles_to_prompts([p.prompt], [p.negative_prompt], p.styles, [p.seed]) + p.task_args['prompt'] = p.all_prompts[0] + p.task_args['negative_prompt'] = p.all_negative_prompts[0] + + # w = shared.sd_model.transformer.config.sample_width * shared.sd_model.vae_scale_factor_spatial + # h = shared.sd_model.transformer.config.sample_height * shared.sd_model.vae_scale_factor_spatial + # n = shared.sd_model.transformer.config.sample_frames * shared.sd_model.vae_scale_factor_temporal + + # run processing + t0 = time.time() + shared.state.disable_preview = True + shared.log.debug(f'Video: cls={shared.sd_model.__class__.__name__} width={p.width} height={p.height} frames={num_frames}') + processed = processing.process_images(p) + shared.state.disable_preview = False + t1 = time.time() + if processed is not None and len(processed.images) > 0: + shared.log.info(f'Video: frames={len(processed.images)} time={t1-t0:.2f}') + if video_type != 'None': + images.save_video(p, filename=None, images=processed.images, video_type=video_type, duration=duration, loop=gif_loop, pad=mp4_pad, interpolate=mp4_interpolate) + return processed From abaa01baa31972bdb7271ec8536dce64d7d10d34 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 1 Jan 2025 15:48:30 -0500 Subject: [PATCH 212/249] fix vae tiling defaults Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 1 + modules/sd_models.py | 8 +++++--- modules/shared.py | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f49388a6..7ecde280c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ - explict clear caches on model load - lock adetailer commit: `#a89c01d` - xyzgrid fix progress calculation + - vae tiling use default value if not set ## Update for 2024-12-31 diff --git a/modules/sd_models.py b/modules/sd_models.py index 46463a85a..c401ae6b7 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -246,9 +246,11 @@ def set_vae_options(sd_model, vae = None, op: str = 'model'): if hasattr(sd_model, "enable_vae_tiling"): if shared.opts.diffusers_vae_tiling: if hasattr(sd_model, 'vae') and hasattr(sd_model.vae, 'config') and hasattr(sd_model.vae.config, 'sample_size') and isinstance(sd_model.vae.config.sample_size, int): - sd_model.vae.tile_sample_min_size = int(shared.opts.diffusers_vae_tile_size) - sd_model.vae.tile_latent_min_size = int(sd_model.vae.config.sample_size / (2 ** (len(sd_model.vae.config.block_out_channels) - 1))) - sd_model.vae.tile_overlap_factor = float(shared.opts.diffusers_vae_tile_overlap) + if shared.opts.diffusers_vae_tile_size > 0: + sd_model.vae.tile_sample_min_size = int(shared.opts.diffusers_vae_tile_size) + sd_model.vae.tile_latent_min_size = int(sd_model.vae.config.sample_size / (2 ** (len(sd_model.vae.config.block_out_channels) - 1))) + if shared.opts.diffusers_vae_tile_overlap != 0.25: + sd_model.vae.tile_overlap_factor = float(shared.opts.diffusers_vae_tile_overlap) shared.log.debug(f'Setting {op}: component=VAE tiling=True tile={sd_model.vae.tile_sample_min_size} overlap={sd_model.vae.tile_overlap_factor}') else: shared.log.debug(f'Setting {op}: component=VAE tiling=True') diff --git a/modules/shared.py b/modules/shared.py index a02c01abf..c7022f36b 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -505,7 +505,7 @@ def get_default_modes(): "no_half_vae": OptionInfo(False if not cmd_opts.use_openvino else True, "Full precision (--no-half-vae)"), "diffusers_vae_slicing": OptionInfo(True, "VAE slicing", gr.Checkbox, {"visible": native}), "diffusers_vae_tiling": OptionInfo(cmd_opts.lowvram or cmd_opts.medvram, "VAE tiling", gr.Checkbox, {"visible": native}), - "diffusers_vae_tile_size": OptionInfo(1024, "VAE tile size", gr.Slider, {"minimum": 256, "maximum": 4096, "step": 8 }), + "diffusers_vae_tile_size": OptionInfo(0, "VAE tile size", gr.Slider, {"minimum": 0, "maximum": 4096, "step": 8 }), "diffusers_vae_tile_overlap": OptionInfo(0.25, "VAE tile overlap", gr.Slider, {"minimum": 0, "maximum": 0.95, "step": 0.05 }), "sd_vae_sliced_encode": OptionInfo(False, "VAE sliced encode", gr.Checkbox, {"visible": not native}), "nan_skip": OptionInfo(False, "Skip Generation if NaN found in latents", gr.Checkbox), From e1553124b3e0e486583d3bf5e8b8c6e863a93e9a Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 1 Jan 2025 16:03:42 -0500 Subject: [PATCH 213/249] update readme Signed-off-by: Vladimir Mandic --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a612f6127..e9312568e 100644 --- a/README.md +++ b/README.md @@ -31,8 +31,9 @@ All individual features are not listed here, instead check [ChangeLog](CHANGELOG - Multiplatform! ▹ **Windows | Linux | MacOS | nVidia | AMD | IntelArc/IPEX | DirectML | OpenVINO | ONNX+Olive | ZLUDA** - Platform specific autodetection and tuning performed on install -- Optimized processing with latest `torch` developments with built-in support for `torch.compile` - and multiple compile backends: *Triton, StableFast, DeepCache, NNCF, OneDiff* +- Optimized processing with latest `torch` developments with built-in support for model compile, quantize and compress + Compile backends: *Triton | StableFast | DeepCache | OneDiff* + Quantization and compression methods: *BitsAndBytes | TorchAO | Optimum-Quanto | NNCF* - Built-in queue management - Built in installer with automatic updates and dependency management - Mobile compatible From 865dcd9080cf86daa663227cd6d1c70797f37a4a Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 2 Jan 2025 11:00:10 -0500 Subject: [PATCH 214/249] fix xyz with detailer, sd35 img2img Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 7 +++++-- modules/postprocess/yolo.py | 24 +++++++++++------------- modules/processing.py | 2 +- modules/processing_args.py | 3 +++ modules/processing_class.py | 2 +- modules/sd_models.py | 2 +- modules/sd_samplers_diffusers.py | 5 +++++ scripts/xyz_grid_on.py | 2 ++ 8 files changed, 29 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ecde280c..8a24d8ff5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Change Log for SD.Next -## Update for 2025-01-01 +## Update for 2025-01-02 - [Allegro Video](https://huggingface.co/rhymes-ai/Allegro) - optimizations: full offload and quantization support @@ -18,8 +18,11 @@ - **Fixes**: - explict clear caches on model load - lock adetailer commit: `#a89c01d` - - xyzgrid fix progress calculation + - xyzgrid progress calculation + - xyzgrid detailer - vae tiling use default value if not set + - sd35 img2img + - samplers test for scale noise before using ## Update for 2024-12-31 diff --git a/modules/postprocess/yolo.py b/modules/postprocess/yolo.py index 3ee2406d1..4011147ec 100644 --- a/modules/postprocess/yolo.py +++ b/modules/postprocess/yolo.py @@ -1,5 +1,6 @@ from typing import TYPE_CHECKING import os +from copy import copy import numpy as np import gradio as gr from PIL import Image, ImageDraw @@ -259,23 +260,23 @@ def restore(self, np_image, p: processing.StableDiffusionProcessing = None): report = [{'label': i.label, 'score': i.score, 'size': f'{i.width}x{i.height}' } for i in items] shared.log.info(f'Detailer: model="{name}" items={report} args={items[0].args} denoise={p.denoising_strength} blur={p.mask_blur} width={p.width} height={p.height} padding={p.inpaint_full_res_padding}') - shared.log.debug(f'Detailer: prompt="{prompt}" negative="{negative}"') + # shared.log.debug(f'Detailer: prompt="{prompt}" negative="{negative}"') models_used.append(name) mask_all = [] p.state = '' prev_state = shared.state.job + pc = copy(p) for item in items: if item.mask is None: continue - p.init_images = [image] - p.image_mask = [item.mask] - # mask_all.append(item.mask) - p.recursion = True + pc.init_images = [image] + pc.image_mask = [item.mask] + pc.overlay_images = [] + pc.recursion = True shared.state.job = 'Detailer' - pp = processing.process_images_inner(p) - del p.recursion - p.overlay_images = None # skip applying overlay twice + pp = processing.process_images_inner(pc) + del pc.recursion if pp is not None and pp.images is not None and len(pp.images) > 0: image = pp.images[0] # update image to be reused for next item if len(pp.images) > 1: @@ -298,13 +299,10 @@ def restore(self, np_image, p: processing.StableDiffusionProcessing = None): if len(mask_all) > 0 and shared.opts.include_mask: from modules.control.util import blend p.image_mask = blend([np.array(m) for m in mask_all]) - # combined = blend([np_image, p.image_mask]) - # combined = Image.fromarray(combined) - # combined.save('/tmp/item.png') p.image_mask = Image.fromarray(p.image_mask) - if len(models_used) > 0: - shared.log.debug(f'Detailer processed: models={models_used}') + # if len(models_used) > 0: + # shared.log.debug(f'Detailer processed: models={models_used}') return np_image def ui(self, tab: str): diff --git a/modules/processing.py b/modules/processing.py index c85938268..23fde5dee 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -463,7 +463,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: ipadapter.unapply(shared.sd_model, unload=getattr(p, 'ip_adapter_unload', False)) if shared.opts.include_mask: - if shared.opts.mask_apply_overlay and p.overlay_images is not None and len(p.overlay_images): + if shared.opts.mask_apply_overlay and p.overlay_images is not None and len(p.overlay_images) > 0: p.image_mask = create_binary_mask(p.overlay_images[0]) p.image_mask = ImageOps.invert(p.image_mask) output_images.append(p.image_mask) diff --git a/modules/processing_args.py b/modules/processing_args.py index f39537538..360928de7 100644 --- a/modules/processing_args.py +++ b/modules/processing_args.py @@ -307,6 +307,9 @@ def set_pipeline_args(p, model, prompts:list, negative_prompts:list, prompts_2:t if isinstance(args['image'], torch.Tensor) or isinstance(args['image'], np.ndarray): args['width'] = 8 * args['image'].shape[-1] args['height'] = 8 * args['image'].shape[-2] + elif isinstance(args['image'][0], torch.Tensor) or isinstance(args['image'][0], np.ndarray): + args['width'] = 8 * args['image'][0].shape[-1] + args['height'] = 8 * args['image'][0].shape[-2] else: args['width'] = 8 * math.ceil(args['image'][0].width / 8) args['height'] = 8 * math.ceil(args['image'][0].height / 8) diff --git a/modules/processing_class.py b/modules/processing_class.py index 96761d262..5960fea76 100644 --- a/modules/processing_class.py +++ b/modules/processing_class.py @@ -502,7 +502,7 @@ def init(self, all_prompts=None, all_seeds=None, all_subseeds=None): image = images.resize_image(self.resize_mode, image, self.width, self.height, upscaler_name=self.resize_name, context=self.resize_context) self.width = image.width self.height = image.height - if self.image_mask is not None and shared.opts.mask_apply_overlay and not hasattr(self, 'xyz'): + if self.image_mask is not None and shared.opts.mask_apply_overlay: image_masked = Image.new('RGBa', (image.width, image.height)) image_to_paste = image.convert("RGBA").convert("RGBa") image_to_mask = ImageOps.invert(self.mask_for_overlay.convert('L')) if self.mask_for_overlay is not None else None diff --git a/modules/sd_models.py b/modules/sd_models.py index c401ae6b7..51d25edfa 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -1288,7 +1288,7 @@ def set_diffuser_pipe(pipe, new_pipe_type): new_pipe.image_encoder = image_encoder if feature_extractor is not None: new_pipe.feature_extractor = feature_extractor - if new_pipe.__class__.__name__ == 'FluxPipeline': + if new_pipe.__class__.__name__ in ['FluxPipeline', 'StableDiffusion3Pipeline']: new_pipe.register_modules(image_encoder = image_encoder) new_pipe.register_modules(feature_extractor = feature_extractor) new_pipe.is_sdxl = getattr(pipe, 'is_sdxl', False) # a1111 compatibility item diff --git a/modules/sd_samplers_diffusers.py b/modules/sd_samplers_diffusers.py index 06459d8a8..6e5f37cd7 100644 --- a/modules/sd_samplers_diffusers.py +++ b/modules/sd_samplers_diffusers.py @@ -271,11 +271,16 @@ def __init__(self, name, constructor, model, **kwargs): sampler = constructor(**self.config) accept_sigmas = "sigmas" in set(inspect.signature(sampler.set_timesteps).parameters.keys()) accepts_timesteps = "timesteps" in set(inspect.signature(sampler.set_timesteps).parameters.keys()) + accept_scale_noise = hasattr(sampler, "scale_noise") debug(f'Sampler: sampler="{name}" sigmas={accept_sigmas} timesteps={accepts_timesteps}') if ('Flux' in model.__class__.__name__) and (not accept_sigmas): shared.log.warning(f'Sampler: sampler="{name}" does not accept sigmas') self.sampler = None return + if ('StableDiffusion3' in model.__class__.__name__) and (not accept_scale_noise): + shared.log.warning(f'Sampler: sampler="{name}" does not implement scale noise') + self.sampler = None + return self.sampler = sampler if name == 'DC Solver': if not hasattr(self.sampler, 'dc_ratios'): diff --git a/scripts/xyz_grid_on.py b/scripts/xyz_grid_on.py index ba23356da..0abd0fa1c 100644 --- a/scripts/xyz_grid_on.py +++ b/scripts/xyz_grid_on.py @@ -325,12 +325,14 @@ def cell(x, y, z, ix, iy, iz): x_opt.apply(pc, x, xs) y_opt.apply(pc, y, ys) z_opt.apply(pc, z, zs) + try: processed = processing.process_images(pc) except Exception as e: shared.log.error(f"XYZ grid: Failed to process image: {e}") errors.display(e, 'XYZ grid') processed = None + if ix == 0 and iy == 0: # create subgrid info text pc.extra_generation_params = copy(pc.extra_generation_params) pc.extra_generation_params['Script'] = self.title() From 28d9302ccabd0eec8b613e2d3fbc5ace335f80c8 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 2 Jan 2025 16:45:08 -0500 Subject: [PATCH 215/249] fix scheduler api Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 3 +++ extensions-builtin/sd-extension-system-info | 2 +- modules/api/api.py | 4 ++++ modules/api/models.py | 10 +++------- modules/shared.py | 8 ++++++-- 5 files changed, 17 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a24d8ff5..55ecccf30 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ - update installer messages - **Detailer**: - add explicit detailer steps setting +- **SysInfo**: + - update to collected data and benchmarks - **Fixes**: - explict clear caches on model load - lock adetailer commit: `#a89c01d` @@ -23,6 +25,7 @@ - vae tiling use default value if not set - sd35 img2img - samplers test for scale noise before using + - scheduler api ## Update for 2024-12-31 diff --git a/extensions-builtin/sd-extension-system-info b/extensions-builtin/sd-extension-system-info index dfa01ce99..aafc660ba 160000 --- a/extensions-builtin/sd-extension-system-info +++ b/extensions-builtin/sd-extension-system-info @@ -1 +1 @@ -Subproject commit dfa01ce99a17d76b45284ef28cef018ff52ac353 +Subproject commit aafc660ba3cf78724059917f7c3a05f3ac1ed46a diff --git a/modules/api/api.py b/modules/api/api.py index b958085ea..b38456d8f 100644 --- a/modules/api/api.py +++ b/modules/api/api.py @@ -99,6 +99,10 @@ def __init__(self, app: FastAPI, queue_lock: Lock): # gallery api gallery.register_api(app) + # compatibility api + self.text2imgapi = self.generate.post_text2img + self.img2imgapi = self.generate.post_img2img + def add_api_route(self, path: str, endpoint, **kwargs): if (shared.cmd_opts.auth or shared.cmd_opts.auth_file) and shared.cmd_opts.api_only: return self.app.add_api_route(path, endpoint, dependencies=[Depends(self.auth)], **kwargs) diff --git a/modules/api/models.py b/modules/api/models.py index 39bcbe383..ce423b639 100644 --- a/modules/api/models.py +++ b/modules/api/models.py @@ -1,5 +1,5 @@ import inspect -from typing import Any, Optional, Dict, List, Type, Callable +from typing import Any, Optional, Dict, List, Type, Callable, Union from pydantic import BaseModel, Field, create_model # pylint: disable=no-name-in-module from inflection import underscore from modules.processing import StableDiffusionProcessingTxt2Img, StableDiffusionProcessingImg2Img @@ -194,7 +194,7 @@ class ItemExtension(BaseModel): "StableDiffusionProcessingTxt2Img", StableDiffusionProcessingTxt2Img, [ - {"key": "sampler_index", "type": int, "default": 0}, + {"key": "sampler_index", "type": Union[int, str], "default": 0}, {"key": "sampler_name", "type": str, "default": "UniPC"}, {"key": "hr_sampler_name", "type": str, "default": "Same as primary"}, {"key": "script_name", "type": str, "default": "none"}, @@ -218,7 +218,7 @@ class ResTxt2Img(BaseModel): "StableDiffusionProcessingImg2Img", StableDiffusionProcessingImg2Img, [ - {"key": "sampler_index", "type": int, "default": 0}, + {"key": "sampler_index", "type": Union[int, str], "default": 0}, {"key": "sampler_name", "type": str, "default": "UniPC"}, {"key": "hr_sampler_name", "type": str, "default": "Same as primary"}, {"key": "script_name", "type": str, "default": "none"}, @@ -405,10 +405,6 @@ class ResNVML(BaseModel): # definition of http response state: str = Field(title="State") -# compatibility items -StableDiffusionTxt2ImgProcessingAPI = ResTxt2Img -StableDiffusionImg2ImgProcessingAPI = ResImg2Img - # helper function def create_model_from_signature(func: Callable, model_name: str, base_model: Type[BaseModel] = BaseModel, additional_fields: List = [], exclude_fields: List[str] = []): diff --git a/modules/shared.py b/modules/shared.py index c7022f36b..afce4266a 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -303,8 +303,12 @@ def validate(self, opt, value): # return False minimum = args.get("minimum", None) maximum = args.get("maximum", None) - if (minimum is not None and value < minimum) or (maximum is not None and value > maximum): - log.error(f'Setting validation: "{opt}"={value} default={self.default} minimum={minimum} maximum={maximum}') + try: + if (minimum is not None and value < minimum) or (maximum is not None and value > maximum): + log.error(f'Setting validation: "{opt}"={value} default={self.default} minimum={minimum} maximum={maximum}') + return False + except Exception as err: + log.error(f'Setting validation: "{opt}"={value} default={self.default} minimum={minimum} maximum={maximum} error={err}') return False return True From 697076bc0b906f774ce0af74f2c7f930445d689b Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 3 Jan 2025 11:55:43 -0500 Subject: [PATCH 216/249] add pixelsmith Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 5 +- modules/pixelsmith/__init__.py | 2 + modules/pixelsmith/autoencoder_kl.py | 496 ++++++ modules/pixelsmith/pixelsmith_pipeline.py | 1882 +++++++++++++++++++++ modules/pixelsmith/vae.py | 983 +++++++++++ modules/sd_models.py | 1 + scripts/pixelsmith.py | 78 + 7 files changed, 3446 insertions(+), 1 deletion(-) create mode 100644 modules/pixelsmith/__init__.py create mode 100644 modules/pixelsmith/autoencoder_kl.py create mode 100644 modules/pixelsmith/pixelsmith_pipeline.py create mode 100644 modules/pixelsmith/vae.py create mode 100644 scripts/pixelsmith.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 55ecccf30..f2bd7d4b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,12 +1,15 @@ # Change Log for SD.Next -## Update for 2025-01-02 +## Update for 2025-01-03 - [Allegro Video](https://huggingface.co/rhymes-ai/Allegro) - optimizations: full offload and quantization support - *reference values*: width 1280 height 720 frames 88 steps 100 guidance 7.5 - *note*: allegro model is really sensitive to input width/height/frames/steps and may result in completely corrupt output if those are not within expected range +- [PixelSmith](https://github.com/Thanos-DB/Pixelsmith/) + - available for SD-XL in txt2img and img2img workflows + - select from *scripts -> pixelsmith* - **Logging**: - reverted enable debug by default - updated [debug wiki](https://github.com/vladmandic/automatic/wiki/debug) diff --git a/modules/pixelsmith/__init__.py b/modules/pixelsmith/__init__.py new file mode 100644 index 000000000..219d65ab8 --- /dev/null +++ b/modules/pixelsmith/__init__.py @@ -0,0 +1,2 @@ +from .pixelsmith_pipeline import PixelSmithXLPipeline +from .autoencoder_kl import PixelSmithVAE diff --git a/modules/pixelsmith/autoencoder_kl.py b/modules/pixelsmith/autoencoder_kl.py new file mode 100644 index 000000000..90c2f9462 --- /dev/null +++ b/modules/pixelsmith/autoencoder_kl.py @@ -0,0 +1,496 @@ +# Original: + +from typing import Dict, Optional, Tuple, Union +import gc +import torch +import torch.nn as nn +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.loaders.single_file_model import FromOriginalModelMixin +from diffusers.utils.accelerate_utils import apply_forward_hook +from diffusers.models.attention_processor import ( + ADDED_KV_ATTENTION_PROCESSORS, + CROSS_ATTENTION_PROCESSORS, + Attention, + AttentionProcessor, + AttnAddedKVProcessor, + AttnProcessor, +) +from diffusers.models.modeling_outputs import AutoencoderKLOutput +from diffusers.models.modeling_utils import ModelMixin +from .vae import Decoder, DecoderOutput, DiagonalGaussianDistribution, Encoder + + +class PixelSmithVAE(ModelMixin, ConfigMixin, FromOriginalModelMixin): + r""" + A VAE model with KL loss for encoding images into latents and decoding latent representations into images. + + This model inherits from [`ModelMixin`]. Check the superclass documentation for it's generic methods implemented + for all models (such as downloading or saving). + + Parameters: + in_channels (int, *optional*, defaults to 3): Number of channels in the input image. + out_channels (int, *optional*, defaults to 3): Number of channels in the output. + down_block_types (`Tuple[str]`, *optional*, defaults to `("DownEncoderBlock2D",)`): + Tuple of downsample block types. + up_block_types (`Tuple[str]`, *optional*, defaults to `("UpDecoderBlock2D",)`): + Tuple of upsample block types. + block_out_channels (`Tuple[int]`, *optional*, defaults to `(64,)`): + Tuple of block output channels. + act_fn (`str`, *optional*, defaults to `"silu"`): The activation function to use. + latent_channels (`int`, *optional*, defaults to 4): Number of channels in the latent space. + sample_size (`int`, *optional*, defaults to `32`): Sample input size. + scaling_factor (`float`, *optional*, defaults to 0.18215): + The component-wise standard deviation of the trained latent space computed using the first batch of the + training set. This is used to scale the latent space to have unit variance when training the diffusion + model. The latents are scaled with the formula `z = z * scaling_factor` before being passed to the + diffusion model. When decoding, the latents are scaled back to the original scale with the formula: `z = 1 + / scaling_factor * z`. For more details, refer to sections 4.3.2 and D.1 of the [High-Resolution Image + Synthesis with Latent Diffusion Models](https://arxiv.org/abs/2112.10752) paper. + force_upcast (`bool`, *optional*, default to `True`): + If enabled it will force the VAE to run in float32 for high image resolution pipelines, such as SD-XL. VAE + can be fine-tuned / trained to a lower range without loosing too much precision in which case + `force_upcast` can be set to `False` - see: https://huggingface.co/madebyollin/sdxl-vae-fp16-fix + """ + + _supports_gradient_checkpointing = True + + @register_to_config + def __init__( + self, + in_channels: int = 3, + out_channels: int = 3, + down_block_types: Tuple[str] = ("DownEncoderBlock2D",), + up_block_types: Tuple[str] = ("UpDecoderBlock2D",), + block_out_channels: Tuple[int] = (64,), + layers_per_block: int = 1, + act_fn: str = "silu", + latent_channels: int = 4, + norm_num_groups: int = 32, + sample_size: int = 32, + scaling_factor: float = 0.18215, + force_upcast: float = True, + ): + super().__init__() + + # pass init params to Encoder + self.encoder = Encoder( + in_channels=in_channels, + out_channels=latent_channels, + down_block_types=down_block_types, + block_out_channels=block_out_channels, + layers_per_block=layers_per_block, + act_fn=act_fn, + norm_num_groups=norm_num_groups, + double_z=True, + ) + + # pass init params to Decoder + self.decoder = Decoder( + in_channels=latent_channels, + out_channels=out_channels, + up_block_types=up_block_types, + block_out_channels=block_out_channels, + layers_per_block=layers_per_block, + norm_num_groups=norm_num_groups, + act_fn=act_fn, + ) + + self.quant_conv = nn.Conv2d(2 * latent_channels, 2 * latent_channels, 1) + self.post_quant_conv = nn.Conv2d(latent_channels, latent_channels, 1) + + self.use_slicing = False + self.use_tiling = False + + # only relevant if vae tiling is enabled + self.tile_sample_min_size = self.config.sample_size + sample_size = ( + self.config.sample_size[0] + if isinstance(self.config.sample_size, (list, tuple)) + else self.config.sample_size + ) + self.tile_latent_min_size = int(sample_size / (2 ** (len(self.config.block_out_channels) - 1))) + self.tile_overlap_factor = 0.25 + + def _set_gradient_checkpointing(self, module, value=False): + if isinstance(module, (Encoder, Decoder)): + module.gradient_checkpointing = value + + def enable_tiling(self, use_tiling: bool = True): + r""" + Enable tiled VAE decoding. When this option is enabled, the VAE will split the input tensor into tiles to + compute decoding and encoding in several steps. This is useful for saving a large amount of memory and to allow + processing larger images. + """ + self.use_tiling = use_tiling + + def disable_tiling(self): + r""" + Disable tiled VAE decoding. If `enable_tiling` was previously enabled, this method will go back to computing + decoding in one step. + """ + self.enable_tiling(False) + + def enable_slicing(self): + r""" + Enable sliced VAE decoding. When this option is enabled, the VAE will split the input tensor in slices to + compute decoding in several steps. This is useful to save some memory and allow larger batch sizes. + """ + self.use_slicing = True + + def disable_slicing(self): + r""" + Disable sliced VAE decoding. If `enable_slicing` was previously enabled, this method will go back to computing + decoding in one step. + """ + self.use_slicing = False + + @property + # Copied from diffusers.models.unet_2d_condition.UNet2DConditionModel.attn_processors + def attn_processors(self) -> Dict[str, AttentionProcessor]: + r""" + Returns: + `dict` of attention processors: A dictionary containing all attention processors used in the model with + indexed by its weight name. + """ + # set recursively + processors = {} + + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + if hasattr(module, "get_processor"): + processors[f"{name}.processor"] = module.get_processor(return_deprecated_lora=True) + + for sub_name, child in module.named_children(): + fn_recursive_add_processors(f"{name}.{sub_name}", child, processors) + + return processors + + for name, module in self.named_children(): + fn_recursive_add_processors(name, module, processors) + + return processors + + # Copied from diffusers.models.unet_2d_condition.UNet2DConditionModel.set_attn_processor + def set_attn_processor( + self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]], _remove_lora=False + ): + r""" + Sets the attention processor to use to compute attention. + + Parameters: + processor (`dict` of `AttentionProcessor` or only `AttentionProcessor`): + The instantiated processor class or a dictionary of processor classes that will be set as the processor + for **all** `Attention` layers. + + If `processor` is a dict, the key needs to define the path to the corresponding cross attention + processor. This is strongly recommended when setting trainable attention processors. + + """ + count = len(self.attn_processors.keys()) + + if isinstance(processor, dict) and len(processor) != count: + raise ValueError( + f"A dict of processors was passed, but the number of processors {len(processor)} does not match the" + f" number of attention layers: {count}. Please make sure to pass {count} processor classes." + ) + + def fn_recursive_attn_processor(name: str, module: torch.nn.Module, processor): + if hasattr(module, "set_processor"): + if not isinstance(processor, dict): + module.set_processor(processor, _remove_lora=_remove_lora) + else: + module.set_processor(processor.pop(f"{name}.processor"), _remove_lora=_remove_lora) + + for sub_name, child in module.named_children(): + fn_recursive_attn_processor(f"{name}.{sub_name}", child, processor) + + for name, module in self.named_children(): + fn_recursive_attn_processor(name, module, processor) + + # Copied from diffusers.models.unet_2d_condition.UNet2DConditionModel.set_default_attn_processor + def set_default_attn_processor(self): + """ + Disables custom attention processors and sets the default attention implementation. + """ + if all(proc.__class__ in ADDED_KV_ATTENTION_PROCESSORS for proc in self.attn_processors.values()): + processor = AttnAddedKVProcessor() + elif all(proc.__class__ in CROSS_ATTENTION_PROCESSORS for proc in self.attn_processors.values()): + processor = AttnProcessor() + else: + raise ValueError( + f"Cannot call `set_default_attn_processor` when attention processors are of type {next(iter(self.attn_processors.values()))}" + ) + + self.set_attn_processor(processor, _remove_lora=True) + + @apply_forward_hook + def encode( + self, x: torch.FloatTensor, return_dict: bool = True + ) -> Union[AutoencoderKLOutput, Tuple[DiagonalGaussianDistribution]]: + """ + Encode a batch of images into latents. + + Args: + x (`torch.FloatTensor`): Input batch of images. + return_dict (`bool`, *optional*, defaults to `True`): + Whether to return a [`~models.autoencoder_kl.AutoencoderKLOutput`] instead of a plain tuple. + + Returns: + The latent representations of the encoded images. If `return_dict` is True, a + [`~models.autoencoder_kl.AutoencoderKLOutput`] is returned, otherwise a plain `tuple` is returned. + """ + if self.use_tiling and (x.shape[-1] > self.tile_sample_min_size or x.shape[-2] > self.tile_sample_min_size): + return self.tiled_encode(x, return_dict=return_dict) + + if self.use_slicing and x.shape[0] > 1: + encoded_slices = [self.encoder(x_slice) for x_slice in x.split(1)] + h = torch.cat(encoded_slices) + else: + h = self.encoder(x) + + moments = self.quant_conv(h) + posterior = DiagonalGaussianDistribution(moments) + + if not return_dict: + return (posterior,) + + return AutoencoderKLOutput(latent_dist=posterior) + + def _decode(self, z: torch.FloatTensor, return_dict: bool = True) -> Union[DecoderOutput, torch.FloatTensor]: + if self.use_tiling and (z.shape[-1] > self.tile_latent_min_size or z.shape[-2] > self.tile_latent_min_size): + return self.tiled_decode(z, return_dict=return_dict) + + z = self.post_quant_conv(z) + dec = self.decoder(z) + + if not return_dict: + return (dec,) + + return DecoderOutput(sample=dec) + + @apply_forward_hook + def decode( + self, z: torch.FloatTensor, return_dict: bool = True, generator=None + ) -> Union[DecoderOutput, torch.FloatTensor]: + """ + Decode a batch of images. + + Args: + z (`torch.FloatTensor`): Input batch of latent vectors. + return_dict (`bool`, *optional*, defaults to `True`): + Whether to return a [`~models.vae.DecoderOutput`] instead of a plain tuple. + + Returns: + [`~models.vae.DecoderOutput`] or `tuple`: + If return_dict is True, a [`~models.vae.DecoderOutput`] is returned, otherwise a plain `tuple` is + returned. + + """ + if self.use_slicing and z.shape[0] > 1: + decoded_slices = [self._decode(z_slice).sample for z_slice in z.split(1)] + decoded = torch.cat(decoded_slices) + else: + decoded = self._decode(z).sample + + if not return_dict: + return (decoded,) + + return DecoderOutput(sample=decoded) + + def blend_v(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int) -> torch.Tensor: + blend_extent = min(a.shape[2], b.shape[2], blend_extent) + for y in range(blend_extent): + b[:, :, y, :] = a[:, :, -blend_extent + y, :] * (1 - y / blend_extent) + b[:, :, y, :] * (y / blend_extent) + return b + + def blend_h(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int) -> torch.Tensor: + blend_extent = min(a.shape[3], b.shape[3], blend_extent) + for x in range(blend_extent): + b[:, :, :, x] = a[:, :, :, -blend_extent + x] * (1 - x / blend_extent) + b[:, :, :, x] * (x / blend_extent) + return b + + def tiled_encode(self, x: torch.FloatTensor, return_dict: bool = True) -> AutoencoderKLOutput: + r"""Encode a batch of images using a tiled encoder. + + When this option is enabled, the VAE will split the input tensor into tiles to compute encoding in several + steps. This is useful to keep memory use constant regardless of image size. The end result of tiled encoding is + different from non-tiled encoding because each tile uses a different encoder. To avoid tiling artifacts, the + tiles overlap and are blended together to form a smooth output. You may still see tile-sized changes in the + output, but they should be much less noticeable. + + Args: + x (`torch.FloatTensor`): Input batch of images. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~models.autoencoder_kl.AutoencoderKLOutput`] instead of a plain tuple. + + Returns: + [`~models.autoencoder_kl.AutoencoderKLOutput`] or `tuple`: + If return_dict is True, a [`~models.autoencoder_kl.AutoencoderKLOutput`] is returned, otherwise a plain + `tuple` is returned. + """ + overlap_size = int(self.tile_sample_min_size * (1 - self.tile_overlap_factor)) + blend_extent = int(self.tile_latent_min_size * self.tile_overlap_factor) + row_limit = self.tile_latent_min_size - blend_extent + + # Split the image into 512x512 tiles and encode them separately. + rows = [] + for i in range(0, x.shape[2], overlap_size): + row = [] + for j in range(0, x.shape[3], overlap_size): + tile = x[:, :, i : i + self.tile_sample_min_size, j : j + self.tile_sample_min_size] + tile = self.encoder(tile.to("cuda")) + tile = self.quant_conv(tile) + row.append(tile) + rows.append(row) + # + del row + gc.collect() + torch.cuda.empty_cache() + # + result_rows = [] + for i, row in enumerate(rows): + result_row = [] + for j, tile in enumerate(row): + # blend the above tile and the left tile + # to the current tile and add the current tile to the result row + if i > 0: + tile = self.blend_v(rows[i - 1][j], tile, blend_extent) + if j > 0: + tile = self.blend_h(row[j - 1], tile, blend_extent) + result_row.append(tile[:, :, :row_limit, :row_limit]) + result_rows.append(torch.cat(result_row, dim=3)) + # + del result_row + gc.collect() + torch.cuda.empty_cache() + # + + moments = torch.cat(result_rows, dim=2) + # + del result_rows + gc.collect() + torch.cuda.empty_cache() + # + posterior = DiagonalGaussianDistribution(moments) + # + del moments + gc.collect() + torch.cuda.empty_cache() + # + if not return_dict: + return (posterior,) + + return AutoencoderKLOutput(latent_dist=posterior) + + def tiled_decode(self, z: torch.FloatTensor, return_dict: bool = True) -> Union[DecoderOutput, torch.FloatTensor]: + r""" + Decode a batch of images using a tiled decoder. + + Args: + z (`torch.FloatTensor`): Input batch of latent vectors. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~models.vae.DecoderOutput`] instead of a plain tuple. + + Returns: + [`~models.vae.DecoderOutput`] or `tuple`: + If return_dict is True, a [`~models.vae.DecoderOutput`] is returned, otherwise a plain `tuple` is + returned. + """ + overlap_size = int(self.tile_latent_min_size * (1 - self.tile_overlap_factor)) + blend_extent = int(self.tile_sample_min_size * self.tile_overlap_factor) + row_limit = self.tile_sample_min_size - blend_extent + + # Split z into overlapping 64x64 tiles and decode them separately. + # The tiles have an overlap to avoid seams between tiles. + rows = [] + for i in range(0, z.shape[2], overlap_size): + row = [] + for j in range(0, z.shape[3], overlap_size): + tile = z[:, :, i : i + self.tile_latent_min_size, j : j + self.tile_latent_min_size] + tile = self.post_quant_conv(tile) + decoded = self.decoder(tile).to("cpu") + row.append(decoded) + rows.append(row) + result_rows = [] + for i, row in enumerate(rows): + result_row = [] + for j, tile in enumerate(row): + # blend the above tile and the left tile + # to the current tile and add the current tile to the result row + if i > 0: + tile = self.blend_v(rows[i - 1][j], tile, blend_extent) + if j > 0: + tile = self.blend_h(row[j - 1], tile, blend_extent) + result_row.append(tile[:, :, :row_limit, :row_limit]) + result_rows.append(torch.cat(result_row, dim=3)) + + dec = torch.cat(result_rows, dim=2) + if not return_dict: + return (dec,) + + return DecoderOutput(sample=dec) + + def forward( + self, + sample: torch.FloatTensor, + sample_posterior: bool = False, + return_dict: bool = True, + generator: Optional[torch.Generator] = None, + ) -> Union[DecoderOutput, torch.FloatTensor]: + r""" + Args: + sample (`torch.FloatTensor`): Input sample. + sample_posterior (`bool`, *optional*, defaults to `False`): + Whether to sample from the posterior. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`DecoderOutput`] instead of a plain tuple. + """ + x = sample + posterior = self.encode(x).latent_dist + if sample_posterior: + z = posterior.sample(generator=generator) + else: + z = posterior.mode() + dec = self.decode(z).sample + + if not return_dict: + return (dec,) + + return DecoderOutput(sample=dec) + + # Copied from diffusers.models.unet_2d_condition.UNet2DConditionModel.fuse_qkv_projections + def fuse_qkv_projections(self): + """ + Enables fused QKV projections. For self-attention modules, all projection matrices (i.e., query, + key, value) are fused. For cross-attention modules, key and value projection matrices are fused. + + + + This API is 🧪 experimental. + + + """ + self.original_attn_processors = None + + for _, attn_processor in self.attn_processors.items(): + if "Added" in str(attn_processor.__class__.__name__): + raise ValueError("`fuse_qkv_projections()` is not supported for models having added KV projections.") + + self.original_attn_processors = self.attn_processors + + for module in self.modules(): + if isinstance(module, Attention): + module.fuse_projections(fuse=True) + + # Copied from diffusers.models.unet_2d_condition.UNet2DConditionModel.unfuse_qkv_projections + def unfuse_qkv_projections(self): + """Disables the fused QKV projection if enabled. + + + + This API is 🧪 experimental. + + + + """ + if self.original_attn_processors is not None: + self.set_attn_processor(self.original_attn_processors) diff --git a/modules/pixelsmith/pixelsmith_pipeline.py b/modules/pixelsmith/pixelsmith_pipeline.py new file mode 100644 index 000000000..4fd21a2fa --- /dev/null +++ b/modules/pixelsmith/pixelsmith_pipeline.py @@ -0,0 +1,1882 @@ +# Original: + +from typing import Any, Dict, List, Optional, Tuple, Union +import inspect +import numpy as np +import matplotlib.pyplot as plt +import torch +import torch.nn.functional as F + +from transformers import ( + CLIPImageProcessor, + CLIPTextModel, + CLIPTextModelWithProjection, + CLIPTokenizer, + CLIPVisionModelWithProjection, +) +from diffusers.image_processor import PipelineImageInput, VaeImageProcessor +from diffusers.loaders import ( + FromSingleFileMixin, + IPAdapterMixin, + StableDiffusionXLLoraLoaderMixin, + TextualInversionLoaderMixin, +) +from diffusers.models import ImageProjection, UNet2DConditionModel +from diffusers.models.attention_processor import ( + Attention, + AttnProcessor2_0, + FusedAttnProcessor2_0, + LoRAAttnProcessor2_0, + LoRAXFormersAttnProcessor, + XFormersAttnProcessor, +) +from diffusers.models.lora import adjust_lora_scale_text_encoder +from diffusers.schedulers import KarrasDiffusionSchedulers +from diffusers.utils import ( + USE_PEFT_BACKEND, + deprecate, + is_torch_xla_available, + logging, + replace_example_docstring, + scale_lora_layers, + unscale_lora_layers, +) +from diffusers.utils.torch_utils import randn_tensor +from diffusers.pipelines.pipeline_utils import DiffusionPipeline, ImagePipelineOutput +from .autoencoder_kl import PixelSmithVAE + + +if is_torch_xla_available(): + import torch_xla.core.xla_model as xm + XLA_AVAILABLE = True +else: + XLA_AVAILABLE = False + + +plt.rcParams['figure.dpi'] = 300 +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> from diffusers import StableDiffusionXLPipeline + + >>> pipe = StableDiffusionXLPipeline.from_pretrained( + ... "stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16 + ... ) + >>> pipe = pipe.to("cuda") + + >>> prompt = "a photo of an astronaut riding a horse on mars" + >>> image = pipe(prompt).images[0] + ``` +""" +#+# +class PAGIdentitySelfAttnProcessor: + r""" + Processor for implementing scaled dot-product attention (enabled by default if you're using PyTorch 2.0). + """ + + def __init__(self): + if not hasattr(F, "scaled_dot_product_attention"): + raise ImportError("AttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.") + + def __call__( + self, + attn: Attention, + hidden_states: torch.FloatTensor, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + temb: Optional[torch.FloatTensor] = None, + *args, + **kwargs, + ) -> torch.FloatTensor: + if len(args) > 0 or kwargs.get("scale", None) is not None: + deprecation_message = "The `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`." + deprecate("scale", "1.0.0", deprecation_message) + + residual = hidden_states + if attn.spatial_norm is not None: + hidden_states = attn.spatial_norm(hidden_states, temb) + + input_ndim = hidden_states.ndim + if input_ndim == 4: + batch_size, channel, height, width = hidden_states.shape + hidden_states = hidden_states.view(batch_size, channel, height * width).transpose(1, 2) + + # chunk + hidden_states_org, hidden_states_ptb = hidden_states.chunk(2) + + # original path + batch_size, sequence_length, _ = hidden_states_org.shape + + if attention_mask is not None: + attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size) + # scaled_dot_product_attention expects attention_mask shape to be + # (batch, heads, source_length, target_length) + attention_mask = attention_mask.view(batch_size, attn.heads, -1, attention_mask.shape[-1]) + + if attn.group_norm is not None: + hidden_states_org = attn.group_norm(hidden_states_org.transpose(1, 2)).transpose(1, 2) + + query = attn.to_q(hidden_states_org) + key = attn.to_k(hidden_states_org) + value = attn.to_v(hidden_states_org) + + inner_dim = key.shape[-1] + head_dim = inner_dim // attn.heads + + query = query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) + + key = key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) + value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) + + # the output of sdp = (batch, num_heads, seq_len, head_dim) + # TODO: add support for attn.scale when we move to Torch 2.1 + hidden_states_org = F.scaled_dot_product_attention( + query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False + ) + + hidden_states_org = hidden_states_org.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim) + hidden_states_org = hidden_states_org.to(query.dtype) + + # linear proj + hidden_states_org = attn.to_out[0](hidden_states_org) + # dropout + hidden_states_org = attn.to_out[1](hidden_states_org) + + if input_ndim == 4: + hidden_states_org = hidden_states_org.transpose(-1, -2).reshape(batch_size, channel, height, width) + + # perturbed path (identity attention) + batch_size, sequence_length, _ = hidden_states_ptb.shape + + if attention_mask is not None: + attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size) + # scaled_dot_product_attention expects attention_mask shape to be + # (batch, heads, source_length, target_length) + attention_mask = attention_mask.view(batch_size, attn.heads, -1, attention_mask.shape[-1]) + + if attn.group_norm is not None: + hidden_states_ptb = attn.group_norm(hidden_states_ptb.transpose(1, 2)).transpose(1, 2) + + value = attn.to_v(hidden_states_ptb) + + # hidden_states_ptb = torch.zeros(value.shape).to(value.get_device()) + hidden_states_ptb = value + + hidden_states_ptb = hidden_states_ptb.to(query.dtype) + + # linear proj + hidden_states_ptb = attn.to_out[0](hidden_states_ptb) + # dropout + hidden_states_ptb = attn.to_out[1](hidden_states_ptb) + + if input_ndim == 4: + hidden_states_ptb = hidden_states_ptb.transpose(-1, -2).reshape(batch_size, channel, height, width) + + # cat + hidden_states = torch.cat([hidden_states_org, hidden_states_ptb]) + + if attn.residual_connection: + hidden_states = hidden_states + residual + + hidden_states = hidden_states / attn.rescale_output_factor + + return hidden_states + + +class PAGCFGIdentitySelfAttnProcessor: + r""" + Processor for implementing scaled dot-product attention (enabled by default if you're using PyTorch 2.0). + """ + + def __init__(self): + if not hasattr(F, "scaled_dot_product_attention"): + raise ImportError("AttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.") + + def __call__( + self, + attn: Attention, + hidden_states: torch.FloatTensor, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + temb: Optional[torch.FloatTensor] = None, + *args, + **kwargs, + ) -> torch.FloatTensor: + if len(args) > 0 or kwargs.get("scale", None) is not None: + deprecation_message = "The `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`." + deprecate("scale", "1.0.0", deprecation_message) + + residual = hidden_states + if attn.spatial_norm is not None: + hidden_states = attn.spatial_norm(hidden_states, temb) + + input_ndim = hidden_states.ndim + if input_ndim == 4: + batch_size, channel, height, width = hidden_states.shape + hidden_states = hidden_states.view(batch_size, channel, height * width).transpose(1, 2) + + # chunk + hidden_states_uncond, hidden_states_org, hidden_states_ptb = hidden_states.chunk(3) + hidden_states_org = torch.cat([hidden_states_uncond, hidden_states_org]) + + # original path + batch_size, sequence_length, _ = hidden_states_org.shape + + if attention_mask is not None: + attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size) + # scaled_dot_product_attention expects attention_mask shape to be + # (batch, heads, source_length, target_length) + attention_mask = attention_mask.view(batch_size, attn.heads, -1, attention_mask.shape[-1]) + + if attn.group_norm is not None: + hidden_states_org = attn.group_norm(hidden_states_org.transpose(1, 2)).transpose(1, 2) + + query = attn.to_q(hidden_states_org) + key = attn.to_k(hidden_states_org) + value = attn.to_v(hidden_states_org) + + inner_dim = key.shape[-1] + head_dim = inner_dim // attn.heads + + query = query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) + + key = key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) + value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) + + # the output of sdp = (batch, num_heads, seq_len, head_dim) + # TODO: add support for attn.scale when we move to Torch 2.1 + hidden_states_org = F.scaled_dot_product_attention( + query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False + ) + + hidden_states_org = hidden_states_org.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim) + hidden_states_org = hidden_states_org.to(query.dtype) + + # linear proj + hidden_states_org = attn.to_out[0](hidden_states_org) + # dropout + hidden_states_org = attn.to_out[1](hidden_states_org) + + if input_ndim == 4: + hidden_states_org = hidden_states_org.transpose(-1, -2).reshape(batch_size, channel, height, width) + + # perturbed path (identity attention) + batch_size, sequence_length, _ = hidden_states_ptb.shape + + if attention_mask is not None: + attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size) + # scaled_dot_product_attention expects attention_mask shape to be + # (batch, heads, source_length, target_length) + attention_mask = attention_mask.view(batch_size, attn.heads, -1, attention_mask.shape[-1]) + + if attn.group_norm is not None: + hidden_states_ptb = attn.group_norm(hidden_states_ptb.transpose(1, 2)).transpose(1, 2) + + value = attn.to_v(hidden_states_ptb) + hidden_states_ptb = value + hidden_states_ptb = hidden_states_ptb.to(query.dtype) + + # linear proj + hidden_states_ptb = attn.to_out[0](hidden_states_ptb) + # dropout + hidden_states_ptb = attn.to_out[1](hidden_states_ptb) + + if input_ndim == 4: + hidden_states_ptb = hidden_states_ptb.transpose(-1, -2).reshape(batch_size, channel, height, width) + + # cat + hidden_states = torch.cat([hidden_states_org, hidden_states_ptb]) + + if attn.residual_connection: + hidden_states = hidden_states + residual + + hidden_states = hidden_states / attn.rescale_output_factor + + return hidden_states + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.rescale_noise_cfg +def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0): + """ + Rescale `noise_cfg` according to `guidance_rescale`. Based on findings of [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). See Section 3.4 + """ + std_text = noise_pred_text.std(dim=list(range(1, noise_pred_text.ndim)), keepdim=True) + std_cfg = noise_cfg.std(dim=list(range(1, noise_cfg.ndim)), keepdim=True) + # rescale the results from guidance (fixes overexposure) + noise_pred_rescaled = noise_cfg * (std_text / std_cfg) + # mix with the original results from guidance by factor guidance_rescale to avoid "plain looking" images + noise_cfg = guidance_rescale * noise_pred_rescaled + (1 - guidance_rescale) * noise_cfg + return noise_cfg + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[List[int]] = None, + **kwargs, +): + """ + Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles + custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. + + Args: + scheduler (`SchedulerMixin`): + The scheduler to get timesteps from. + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, + `timesteps` must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to support arbitrary spacing between timesteps. If `None`, then the default + timestep spacing strategy of the scheduler is used. If `timesteps` is passed, `num_inference_steps` + must be `None`. + + Returns: + `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + second element is the number of inference steps. + """ + if timesteps is not None: + accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + + +class PixelSmithXLPipeline( + DiffusionPipeline, + #StableDiffusionMixin, + FromSingleFileMixin, + StableDiffusionXLLoraLoaderMixin, + TextualInversionLoaderMixin, + IPAdapterMixin, +): + r""" + Pipeline for text-to-image generation using Stable Diffusion XL. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + The pipeline also inherits the following loading methods: + - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings + - [`~loaders.FromSingleFileMixin.from_single_file`] for loading `.ckpt` files + - [`~loaders.StableDiffusionXLLoraLoaderMixin.load_lora_weights`] for loading LoRA weights + - [`~loaders.StableDiffusionXLLoraLoaderMixin.save_lora_weights`] for saving LoRA weights + - [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. + text_encoder ([`CLIPTextModel`]): + Frozen text-encoder. Stable Diffusion XL uses the text portion of + [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically + the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant. + text_encoder_2 ([` CLIPTextModelWithProjection`]): + Second frozen text-encoder. Stable Diffusion XL uses the text and pool portion of + [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModelWithProjection), + specifically the + [laion/CLIP-ViT-bigG-14-laion2B-39B-b160k](https://huggingface.co/laion/CLIP-ViT-bigG-14-laion2B-39B-b160k) + variant. + tokenizer (`CLIPTokenizer`): + Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). + tokenizer_2 (`CLIPTokenizer`): + Second Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). + unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + force_zeros_for_empty_prompt (`bool`, *optional*, defaults to `"True"`): + Whether the negative prompt embeddings shall be forced to always be set to 0. Also see the config of + `stabilityai/stable-diffusion-xl-base-1-0`. + add_watermarker (`bool`, *optional*): + Whether to use the [invisible_watermark library](https://github.com/ShieldMnt/invisible-watermark/) to + watermark output images. If not defined, it will default to True if the package is installed, otherwise no + watermarker will be used. + """ + + model_cpu_offload_seq = "text_encoder->text_encoder_2->image_encoder->unet->vae" + _optional_components = [ + "tokenizer", + "tokenizer_2", + "text_encoder", + "text_encoder_2", + "image_encoder", + "feature_extractor", + ] + _callback_tensor_inputs = [ + "latents", + "prompt_embeds", + "negative_prompt_embeds", + "add_text_embeds", + "add_time_ids", + "negative_pooled_prompt_embeds", + "negative_add_time_ids", + ] + + def __init__( + self, + vae: PixelSmithVAE, + text_encoder: CLIPTextModel, + text_encoder_2: CLIPTextModelWithProjection, + tokenizer: CLIPTokenizer, + tokenizer_2: CLIPTokenizer, + unet: UNet2DConditionModel, + scheduler: KarrasDiffusionSchedulers, + image_encoder: CLIPVisionModelWithProjection = None, + feature_extractor: CLIPImageProcessor = None, + force_zeros_for_empty_prompt: bool = True, + add_watermarker: Optional[bool] = None, + ): + super().__init__() + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + text_encoder_2=text_encoder_2, + tokenizer=tokenizer, + tokenizer_2=tokenizer_2, + unet=unet, + scheduler=scheduler, + image_encoder=image_encoder, + feature_extractor=feature_extractor, + ) + self.register_to_config(force_zeros_for_empty_prompt=force_zeros_for_empty_prompt) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + + self.default_sample_size = self.unet.config.sample_size + + def encode_prompt( + self, + prompt: str, + prompt_2: Optional[str] = None, + device: Optional[torch.device] = None, + num_images_per_prompt: int = 1, + do_classifier_free_guidance: bool = True, + negative_prompt: Optional[str] = None, + negative_prompt_2: Optional[str] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + clip_skip: Optional[int] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is + used in both text-encoders + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + negative_prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and + `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. + If not provided, pooled text embeddings will be generated from `prompt` input argument. + negative_pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` + input argument. + lora_scale (`float`, *optional*): + A lora scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + """ + device = device or self._execution_device + + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, StableDiffusionXLLoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if self.text_encoder is not None: + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + if self.text_encoder_2 is not None: + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder_2, lora_scale) + else: + scale_lora_layers(self.text_encoder_2, lora_scale) + + prompt = [prompt] if isinstance(prompt, str) else prompt + + if prompt is not None: + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + # Define tokenizers and text encoders + tokenizers = [self.tokenizer, self.tokenizer_2] if self.tokenizer is not None else [self.tokenizer_2] + text_encoders = ( + [self.text_encoder, self.text_encoder_2] if self.text_encoder is not None else [self.text_encoder_2] + ) + + if prompt_embeds is None: + prompt_2 = prompt_2 or prompt + prompt_2 = [prompt_2] if isinstance(prompt_2, str) else prompt_2 + + # textual inversion: process multi-vector tokens if necessary + prompt_embeds_list = [] + prompts = [prompt, prompt_2] + for prompt, tokenizer, text_encoder in zip(prompts, tokenizers, text_encoders): + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, tokenizer) + + text_inputs = tokenizer( + prompt, + padding="max_length", + max_length=tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + + text_input_ids = text_inputs.input_ids + untruncated_ids = tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = tokenizer.batch_decode(untruncated_ids[:, tokenizer.model_max_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {tokenizer.model_max_length} tokens: {removed_text}" + ) + + prompt_embeds = text_encoder(text_input_ids.to(device), output_hidden_states=True) + + # We are only ALWAYS interested in the pooled output of the final text encoder + pooled_prompt_embeds = prompt_embeds[0] + if clip_skip is None: + prompt_embeds = prompt_embeds.hidden_states[-2] + else: + # "2" because SDXL always indexes from the penultimate layer. + prompt_embeds = prompt_embeds.hidden_states[-(clip_skip + 2)] + + prompt_embeds_list.append(prompt_embeds) + + prompt_embeds = torch.concat(prompt_embeds_list, dim=-1) + + # get unconditional embeddings for classifier free guidance + zero_out_negative_prompt = negative_prompt is None and self.config.force_zeros_for_empty_prompt + if do_classifier_free_guidance and negative_prompt_embeds is None and zero_out_negative_prompt: + negative_prompt_embeds = torch.zeros_like(prompt_embeds) + negative_pooled_prompt_embeds = torch.zeros_like(pooled_prompt_embeds) + elif do_classifier_free_guidance and negative_prompt_embeds is None: + negative_prompt = negative_prompt or "" + negative_prompt_2 = negative_prompt_2 or negative_prompt + + # normalize str to list + negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt + negative_prompt_2 = ( + batch_size * [negative_prompt_2] if isinstance(negative_prompt_2, str) else negative_prompt_2 + ) + + uncond_tokens: List[str] + if prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = [negative_prompt, negative_prompt_2] + + negative_prompt_embeds_list = [] + for negative_prompt, tokenizer, text_encoder in zip(uncond_tokens, tokenizers, text_encoders): + if isinstance(self, TextualInversionLoaderMixin): + negative_prompt = self.maybe_convert_prompt(negative_prompt, tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = tokenizer( + negative_prompt, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + negative_prompt_embeds = text_encoder( + uncond_input.input_ids.to(device), + output_hidden_states=True, + ) + # We are only ALWAYS interested in the pooled output of the final text encoder + negative_pooled_prompt_embeds = negative_prompt_embeds[0] + negative_prompt_embeds = negative_prompt_embeds.hidden_states[-2] + + negative_prompt_embeds_list.append(negative_prompt_embeds) + + negative_prompt_embeds = torch.concat(negative_prompt_embeds_list, dim=-1) + + if self.text_encoder_2 is not None: + prompt_embeds = prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device) + else: + prompt_embeds = prompt_embeds.to(dtype=self.unet.dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + if self.text_encoder_2 is not None: + negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device) + else: + negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.unet.dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + pooled_prompt_embeds = pooled_prompt_embeds.repeat(1, num_images_per_prompt).view( + bs_embed * num_images_per_prompt, -1 + ) + if do_classifier_free_guidance: + negative_pooled_prompt_embeds = negative_pooled_prompt_embeds.repeat(1, num_images_per_prompt).view( + bs_embed * num_images_per_prompt, -1 + ) + + if self.text_encoder is not None: + if isinstance(self, StableDiffusionXLLoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder, lora_scale) + + if self.text_encoder_2 is not None: + if isinstance(self, StableDiffusionXLLoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder_2, lora_scale) + + return prompt_embeds, negative_prompt_embeds, pooled_prompt_embeds, negative_pooled_prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_image + def encode_image(self, image, device, num_images_per_prompt, output_hidden_states=None): + dtype = next(self.image_encoder.parameters()).dtype + + if not isinstance(image, torch.Tensor): + image = self.feature_extractor(image, return_tensors="pt").pixel_values + + image = image.to(device=device, dtype=dtype) + if output_hidden_states: + image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] + image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_enc_hidden_states = self.image_encoder( + torch.zeros_like(image), output_hidden_states=True + ).hidden_states[-2] + uncond_image_enc_hidden_states = uncond_image_enc_hidden_states.repeat_interleave( + num_images_per_prompt, dim=0 + ) + return image_enc_hidden_states, uncond_image_enc_hidden_states + else: + image_embeds = self.image_encoder(image).image_embeds + image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_embeds = torch.zeros_like(image_embeds) + + return image_embeds, uncond_image_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_ip_adapter_image_embeds + def prepare_ip_adapter_image_embeds( + self, ip_adapter_image, ip_adapter_image_embeds, device, num_images_per_prompt, do_classifier_free_guidance + ): + if ip_adapter_image_embeds is None: + if not isinstance(ip_adapter_image, list): + ip_adapter_image = [ip_adapter_image] + + if len(ip_adapter_image) != len(self.unet.encoder_hid_proj.image_projection_layers): + raise ValueError( + f"`ip_adapter_image` must have same length as the number of IP Adapters. Got {len(ip_adapter_image)} images and {len(self.unet.encoder_hid_proj.image_projection_layers)} IP Adapters." + ) + + image_embeds = [] + for single_ip_adapter_image, image_proj_layer in zip( + ip_adapter_image, self.unet.encoder_hid_proj.image_projection_layers + ): + output_hidden_state = not isinstance(image_proj_layer, ImageProjection) + single_image_embeds, single_negative_image_embeds = self.encode_image( + single_ip_adapter_image, device, 1, output_hidden_state + ) + single_image_embeds = torch.stack([single_image_embeds] * num_images_per_prompt, dim=0) + single_negative_image_embeds = torch.stack( + [single_negative_image_embeds] * num_images_per_prompt, dim=0 + ) + + if do_classifier_free_guidance: + single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds]) + single_image_embeds = single_image_embeds.to(device) + + image_embeds.append(single_image_embeds) + else: + repeat_dims = [1] + image_embeds = [] + for single_image_embeds in ip_adapter_image_embeds: + if do_classifier_free_guidance: + single_negative_image_embeds, single_image_embeds = single_image_embeds.chunk(2) + single_image_embeds = single_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_image_embeds.shape[1:])) + ) + single_negative_image_embeds = single_negative_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_negative_image_embeds.shape[1:])) + ) + single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds]) + else: + single_image_embeds = single_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_image_embeds.shape[1:])) + ) + image_embeds.append(single_image_embeds) + + return image_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def check_inputs( + self, + prompt, + prompt_2, + height, + width, + callback_steps, + negative_prompt=None, + negative_prompt_2=None, + prompt_embeds=None, + negative_prompt_embeds=None, + pooled_prompt_embeds=None, + negative_pooled_prompt_embeds=None, + ip_adapter_image=None, + ip_adapter_image_embeds=None, + callback_on_step_end_tensor_inputs=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt_2 is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt_2`: {prompt_2} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + elif prompt_2 is not None and (not isinstance(prompt_2, str) and not isinstance(prompt_2, list)): + raise ValueError(f"`prompt_2` has to be of type `str` or `list` but is {type(prompt_2)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + elif negative_prompt_2 is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt_2`: {negative_prompt_2} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + if prompt_embeds is not None and pooled_prompt_embeds is None: + raise ValueError( + "If `prompt_embeds` are provided, `pooled_prompt_embeds` also have to be passed. Make sure to generate `pooled_prompt_embeds` from the same text encoder that was used to generate `prompt_embeds`." + ) + + if negative_prompt_embeds is not None and negative_pooled_prompt_embeds is None: + raise ValueError( + "If `negative_prompt_embeds` are provided, `negative_pooled_prompt_embeds` also have to be passed. Make sure to generate `negative_pooled_prompt_embeds` from the same text encoder that was used to generate `negative_prompt_embeds`." + ) + + if ip_adapter_image is not None and ip_adapter_image_embeds is not None: + raise ValueError( + "Provide either `ip_adapter_image` or `ip_adapter_image_embeds`. Cannot leave both `ip_adapter_image` and `ip_adapter_image_embeds` defined." + ) + + if ip_adapter_image_embeds is not None: + if not isinstance(ip_adapter_image_embeds, list): + raise ValueError( + f"`ip_adapter_image_embeds` has to be of type `list` but is {type(ip_adapter_image_embeds)}" + ) + elif ip_adapter_image_embeds[0].ndim not in [3, 4]: + raise ValueError( + f"`ip_adapter_image_embeds` has to be a list of 3D or 4D tensors but is {ip_adapter_image_embeds[0].ndim}D" + ) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents + def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None): + shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + def _get_add_time_ids( + self, original_size, crops_coords_top_left, target_size, dtype, text_encoder_projection_dim=None + ): + add_time_ids = list(original_size + crops_coords_top_left + target_size) + + passed_add_embed_dim = ( + self.unet.config.addition_time_embed_dim * len(add_time_ids) + text_encoder_projection_dim + ) + expected_add_embed_dim = self.unet.add_embedding.linear_1.in_features + + if expected_add_embed_dim != passed_add_embed_dim: + raise ValueError( + f"Model expects an added time embedding vector of length {expected_add_embed_dim}, but a vector of {passed_add_embed_dim} was created. The model has an incorrect config. Please check `unet.config.time_embedding_type` and `text_encoder_2.config.projection_dim`." + ) + + add_time_ids = torch.tensor([add_time_ids], dtype=dtype) + return add_time_ids + + def upcast_vae(self): + dtype = self.vae.dtype + self.vae.to(dtype=torch.float32) + use_torch_2_0_or_xformers = isinstance( + self.vae.decoder.mid_block.attentions[0].processor, + ( + AttnProcessor2_0, + XFormersAttnProcessor, + LoRAXFormersAttnProcessor, + LoRAAttnProcessor2_0, + FusedAttnProcessor2_0, + ), + ) + # if xformers or torch_2_0 is used attention block does not need + # to be in float32 which can save lots of memory + if use_torch_2_0_or_xformers: + self.vae.post_quant_conv.to(dtype) + self.vae.decoder.conv_in.to(dtype) + self.vae.decoder.mid_block.to(dtype) + + # Copied from diffusers.pipelines.latent_consistency_models.pipeline_latent_consistency_text2img.LatentConsistencyModelPipeline.get_guidance_scale_embedding + def get_guidance_scale_embedding( + self, w: torch.Tensor, embedding_dim: int = 512, dtype: torch.dtype = torch.float32 + ) -> torch.FloatTensor: + """ + See https://github.com/google-research/vdm/blob/dc27b98a554f65cdc654b800da5aa1846545d41b/model_vdm.py#L298 + + Args: + w (`torch.Tensor`): + Generate embedding vectors with a specified guidance scale to subsequently enrich timestep embeddings. + embedding_dim (`int`, *optional*, defaults to 512): + Dimension of the embeddings to generate. + dtype (`torch.dtype`, *optional*, defaults to `torch.float32`): + Data type of the generated embeddings. + + Returns: + `torch.FloatTensor`: Embedding vectors with shape `(len(w), embedding_dim)`. + """ + assert len(w.shape) == 1 + w = w * 1000.0 + + half_dim = embedding_dim // 2 + emb = torch.log(torch.tensor(10000.0)) / (half_dim - 1) + emb = torch.exp(torch.arange(half_dim, dtype=dtype) * -emb) + emb = w.to(dtype)[:, None] * emb[None, :] + emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1) + if embedding_dim % 2 == 1: # zero pad + emb = torch.nn.functional.pad(emb, (0, 1)) + assert emb.shape == (w.shape[0], embedding_dim) + return emb + #+# + + def pred_z0(self, sample, model_output, timestep): + alpha_prod_t = self.scheduler.alphas_cumprod[timestep].to(sample.device) + + beta_prod_t = 1 - alpha_prod_t + if self.scheduler.config.prediction_type == "epsilon": + pred_original_sample = (sample - beta_prod_t ** (0.5) * model_output) / alpha_prod_t ** (0.5) + elif self.scheduler.config.prediction_type == "sample": + pred_original_sample = model_output + elif self.scheduler.config.prediction_type == "v_prediction": + pred_original_sample = (alpha_prod_t**0.5) * sample - (beta_prod_t**0.5) * model_output + # predict V + model_output = (alpha_prod_t**0.5) * model_output + (beta_prod_t**0.5) * sample + else: + raise ValueError( + f"prediction_type given as {self.scheduler.config.prediction_type} must be one of `epsilon`, `sample`," + " or `v_prediction`" + ) + + return pred_original_sample + + def pred_x0(self, latents, noise_pred, t, generator, device, prompt_embeds, output_type): + pred_z0 = self.pred_z0(latents, noise_pred, t) + pred_x0 = self.vae.decode( + pred_z0 / self.vae.config.scaling_factor, + return_dict=False, + generator=generator + )[0] + do_denormalize = [True] * pred_x0.shape[0] + pred_x0 = self.image_processor.postprocess(pred_x0, output_type=output_type, do_denormalize=do_denormalize) + + return pred_x0 + + #+# + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def guidance_rescale(self): + return self._guidance_rescale + + @property + def clip_skip(self): + return self._clip_skip + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + @property + def do_classifier_free_guidance(self): + return self._guidance_scale > 1 and self.unet.config.time_cond_proj_dim is None + + @property + def cross_attention_kwargs(self): + return self._cross_attention_kwargs + + @property + def denoising_end(self): + return self._denoising_end + + @property + def num_timesteps(self): + return self._num_timesteps + + @property + def interrupt(self): + return self._interrupt + + #+# + + @property + def pag_scale(self): + return self._pag_scale + + @property + def do_adversarial_guidance(self): + return self._pag_scale > 0 + + @property + def pag_adaptive_scaling(self): + return self._pag_adaptive_scaling + + @property + def do_pag_adaptive_scaling(self): + return self._pag_adaptive_scaling > 0 + + @property + def pag_drop_rate(self): + return self._pag_drop_rate + + @property + def pag_applied_layers(self): + return self._pag_applied_layers + + @property + def pag_applied_layers_index(self): + return self._pag_applied_layers_index + #+# + + def _random_crop(self, z, i, j, patch_size): + p=patch_size//2 + return z[...,i-p:i+p, j-p:j+p] + + def get_value_coordinates(self, tensor): + value_indices = torch.nonzero(tensor == tensor.max(), as_tuple=False) + random_indices = value_indices[torch.randperm(value_indices.size(0))] + return random_indices + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + prompt_2: Optional[Union[str, List[str]]] = None, + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 50, + timesteps: List[int] = None, + denoising_end: Optional[float] = None, + guidance_scale: float = 5.0, + #+# + pag_scale: float = 0.0, # longer inference time if used (https://ku-cvlab.github.io/Perturbed-Attention-Guidance/) + pag_adaptive_scaling: float = 0.0, + pag_drop_rate: float = 0.5, + pag_applied_layers: List[str] = ['mid'], #['down', 'mid', 'up'] + pag_applied_layers_index: List[str] = None, #['d4', 'd5', 'm0'] + #+# + negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt_2: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + ip_adapter_image: Optional[PipelineImageInput] = None, + ip_adapter_image_embeds: Optional[List[torch.FloatTensor]] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + guidance_rescale: float = 0.0, + original_size: Optional[Tuple[int, int]] = None, + crops_coords_top_left: Tuple[int, int] = (0, 0), + target_size: Optional[Tuple[int, int]] = None, + negative_original_size: Optional[Tuple[int, int]] = None, + negative_crops_coords_top_left: Tuple[int, int] = (0, 0), + negative_target_size: Optional[Tuple[int, int]] = None, + clip_skip: Optional[int] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + image = None, + slider = None, + **kwargs, + ): + r""" + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. + instead. + prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is + used in both text-encoders + height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The height in pixels of the generated image. This is set to 1024 by default for the best results. + Anything below 512 pixels won't work well for + [stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0) + and checkpoints that are not specifically fine-tuned on low resolutions. + width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The width in pixels of the generated image. This is set to 1024 by default for the best results. + Anything below 512 pixels won't work well for + [stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0) + and checkpoints that are not specifically fine-tuned on low resolutions. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + timesteps (`List[int]`, *optional*): + Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument + in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is + passed will be used. Must be in descending order. + denoising_end (`float`, *optional*): + When specified, determines the fraction (between 0.0 and 1.0) of the total denoising process to be + completed before it is intentionally prematurely terminated. As a result, the returned sample will + still retain a substantial amount of noise as determined by the discrete timesteps selected by the + scheduler. The denoising_end parameter should ideally be utilized when this pipeline forms a part of a + "Mixture of Denoisers" multi-pipeline setup, as elaborated in [**Refining the Image + Output**](https://huggingface.co/docs/diffusers/api/pipelines/stable_diffusion/stable_diffusion_xl#refining-the-image-output) + guidance_scale (`float`, *optional*, defaults to 5.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + negative_prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and + `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to + [`schedulers.DDIMScheduler`], will be ignored for others. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. + If not provided, pooled text embeddings will be generated from `prompt` input argument. + negative_pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` + input argument. + ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. + ip_adapter_image_embeds (`List[torch.FloatTensor]`, *optional*): + Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. + Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding + if `do_classifier_free_guidance` is set to `True`. + If not provided, embeddings are computed from the `ip_adapter_image` input argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] instead + of a plain tuple. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + guidance_rescale (`float`, *optional*, defaults to 0.0): + Guidance rescale factor proposed by [Common Diffusion Noise Schedules and Sample Steps are + Flawed](https://arxiv.org/pdf/2305.08891.pdf) `guidance_scale` is defined as `φ` in equation 16. of + [Common Diffusion Noise Schedules and Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). + Guidance rescale factor should fix overexposure when using zero terminal SNR. + original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled. + `original_size` defaults to `(height, width)` if not specified. Part of SDXL's micro-conditioning as + explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position + `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting + `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + For most cases, `target_size` should be set to the desired height and width of the generated image. If + not specified it will default to `(height, width)`. Part of SDXL's micro-conditioning as explained in + section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + negative_original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + To negatively condition the generation process based on a specific image resolution. Part of SDXL's + micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more + information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. + negative_crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + To negatively condition the generation process based on a specific crop coordinates. Part of SDXL's + micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more + information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. + negative_target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + To negatively condition the generation process based on a target image resolution. It should be as same + as the `target_size` for most cases. Part of SDXL's micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more + information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeline class. + image('pil'): + Upscaled image from previous step + slider('int'): + Freedom of the model to be more generative or closer to the input + + Examples: + + Returns: + [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] or `tuple`: + [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] if `return_dict` is True, otherwise a + `tuple`. When returning a tuple, the first element is a list with the generated images. + """ + + callback = kwargs.pop("callback", None) + callback_steps = kwargs.pop("callback_steps", None) + + if callback is not None: + deprecate( + "callback", + "1.0.0", + "Passing `callback` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`", + ) + if callback_steps is not None: + deprecate( + "callback_steps", + "1.0.0", + "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`", + ) + + # 0. Default height and width to unet + height = height or self.default_sample_size * self.vae_scale_factor + width = width or self.default_sample_size * self.vae_scale_factor + + original_size = original_size or (height, width) + target_size = target_size or (height, width) + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + prompt_2, + height, + width, + callback_steps, + negative_prompt, + negative_prompt_2, + prompt_embeds, + negative_prompt_embeds, + pooled_prompt_embeds, + negative_pooled_prompt_embeds, + ip_adapter_image, + ip_adapter_image_embeds, + callback_on_step_end_tensor_inputs, + ) + + self._guidance_scale = guidance_scale + self._guidance_rescale = guidance_rescale + self._clip_skip = clip_skip + self._cross_attention_kwargs = cross_attention_kwargs + self._denoising_end = denoising_end + self._interrupt = False + + #+# + self._pag_scale = pag_scale + self._pag_adaptive_scaling = pag_adaptive_scaling + self._pag_drop_rate = pag_drop_rate + self._pag_applied_layers = pag_applied_layers + self._pag_applied_layers_index = pag_applied_layers_index + #+# + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + + # 3. Encode input prompt + lora_scale = ( + self.cross_attention_kwargs.get("scale", None) if self.cross_attention_kwargs is not None else None + ) + + ( + prompt_embeds, + negative_prompt_embeds, + pooled_prompt_embeds, + negative_pooled_prompt_embeds, + ) = self.encode_prompt( + prompt=prompt, + prompt_2=prompt_2, + device=device, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=self.do_classifier_free_guidance, + negative_prompt=negative_prompt, + negative_prompt_2=negative_prompt_2, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + pooled_prompt_embeds=pooled_prompt_embeds, + negative_pooled_prompt_embeds=negative_pooled_prompt_embeds, + lora_scale=lora_scale, + clip_skip=self.clip_skip, + ) + + # 4. Prepare timesteps + timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, timesteps) + + # 5. Prepare latent variables + num_channels_latents = self.unet.config.in_channels + latents = self.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + ) + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 7. Prepare added time ids & embeddings + add_text_embeds = pooled_prompt_embeds + if self.text_encoder_2 is None: + text_encoder_projection_dim = int(pooled_prompt_embeds.shape[-1]) + else: + text_encoder_projection_dim = self.text_encoder_2.config.projection_dim + + add_time_ids = self._get_add_time_ids( + original_size, + crops_coords_top_left, + target_size, + dtype=prompt_embeds.dtype, + text_encoder_projection_dim=text_encoder_projection_dim, + ) + if negative_original_size is not None and negative_target_size is not None: + negative_add_time_ids = self._get_add_time_ids( + negative_original_size, + negative_crops_coords_top_left, + negative_target_size, + dtype=prompt_embeds.dtype, + text_encoder_projection_dim=text_encoder_projection_dim, + ) + else: + negative_add_time_ids = add_time_ids + + if self.do_classifier_free_guidance and not self.do_adversarial_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0) + add_text_embeds = torch.cat([negative_pooled_prompt_embeds, add_text_embeds], dim=0) + add_time_ids = torch.cat([negative_add_time_ids, add_time_ids], dim=0) + #pag + elif not self.do_classifier_free_guidance and self.do_adversarial_guidance: + prompt_embeds = torch.cat([prompt_embeds, prompt_embeds], dim=0) + add_text_embeds = torch.cat([add_text_embeds, add_text_embeds], dim=0) + add_time_ids = torch.cat([add_time_ids, add_time_ids], dim=0) + #both + elif self.do_classifier_free_guidance and self.do_adversarial_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds, prompt_embeds], dim=0) + add_text_embeds = torch.cat([negative_pooled_prompt_embeds, add_text_embeds, add_text_embeds], dim=0) + add_time_ids = torch.cat([negative_add_time_ids, add_time_ids, add_time_ids], dim=0) + #+# + + prompt_embeds = prompt_embeds.to(device) + add_text_embeds = add_text_embeds.to(device) + add_time_ids = add_time_ids.to(device).repeat(batch_size * num_images_per_prompt, 1) + + if ip_adapter_image is not None or ip_adapter_image_embeds is not None: + image_embeds = self.prepare_ip_adapter_image_embeds( + ip_adapter_image, + ip_adapter_image_embeds, + device, + batch_size * num_images_per_prompt, + self.do_classifier_free_guidance, + ) + + # 8. Denoising loop + num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0) + + # 8.1 Apply denoising_end + if ( + self.denoising_end is not None + and isinstance(self.denoising_end, float) + and self.denoising_end > 0 + and self.denoising_end < 1 + ): + discrete_timestep_cutoff = int( + round( + self.scheduler.config.num_train_timesteps + - (self.denoising_end * self.scheduler.config.num_train_timesteps) + ) + ) + num_inference_steps = len(list(filter(lambda ts: ts >= discrete_timestep_cutoff, timesteps))) + timesteps = timesteps[:num_inference_steps] + + # 9. Optionally get Guidance Scale Embedding + timestep_cond = None + if self.unet.config.time_cond_proj_dim is not None: + guidance_scale_tensor = torch.tensor(self.guidance_scale - 1).repeat(batch_size * num_images_per_prompt) + timestep_cond = self.get_guidance_scale_embedding( + guidance_scale_tensor, embedding_dim=self.unet.config.time_cond_proj_dim + ).to(device=device, dtype=latents.dtype) + + #+# + # 10. Create down mid and up layer lists + if self.do_adversarial_guidance: + down_layers = [] + mid_layers = [] + up_layers = [] + for name, module in self.unet.named_modules(): + if 'attn1' in name and 'to' not in name: + layer_type = name.split('.')[0].split('_')[0] + if layer_type == 'down': + down_layers.append(module) + elif layer_type == 'mid': + mid_layers.append(module) + elif layer_type == 'up': + up_layers.append(module) + else: + raise ValueError(f"Invalid layer type: {layer_type}") + #+# + + self._num_timesteps = len(timesteps) + if image is None: + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + if self.interrupt: + continue + + #+# + # #cfg + if self.do_classifier_free_guidance and not self.do_adversarial_guidance: + latent_model_input = torch.cat([latents] * 2) + #pag + elif not self.do_classifier_free_guidance and self.do_adversarial_guidance: + latent_model_input = torch.cat([latents] * 2) + #both + elif self.do_classifier_free_guidance and self.do_adversarial_guidance: + latent_model_input = torch.cat([latents] * 3) + #no + else: + latent_model_input = latents + + # change attention layer in UNet if use PAG + if self.do_adversarial_guidance: + + if self.do_classifier_free_guidance: + replace_processor = PAGCFGIdentitySelfAttnProcessor() + else: + replace_processor = PAGIdentitySelfAttnProcessor() + + if self.pag_applied_layers_index: + drop_layers = self.pag_applied_layers_index + for drop_layer in drop_layers: + layer_number = int(drop_layer[1:]) + try: + if drop_layer[0] == 'd': + down_layers[layer_number].processor = replace_processor + elif drop_layer[0] == 'm': + mid_layers[layer_number].processor = replace_processor + elif drop_layer[0] == 'u': + up_layers[layer_number].processor = replace_processor + else: + raise ValueError(f"Invalid layer type: {drop_layer[0]}") + except IndexError as err: + raise ValueError(f"Invalid layer index: {drop_layer}. Available layers: {len(down_layers)} down layers, {len(mid_layers)} mid layers, {len(up_layers)} up layers.") from err + elif self.pag_applied_layers: + drop_full_layers = self.pag_applied_layers + for drop_full_layer in drop_full_layers: + try: + if drop_full_layer == "down": + for down_layer in down_layers: + down_layer.processor = replace_processor + elif drop_full_layer == "mid": + for mid_layer in mid_layers: + mid_layer.processor = replace_processor + elif drop_full_layer == "up": + for up_layer in up_layers: + up_layer.processor = replace_processor + else: + raise ValueError(f"Invalid layer type: {drop_full_layer}") + except IndexError as err: + raise ValueError(f"Invalid layer index: {drop_full_layer}. Available layers are: down, mid and up. If you need to specify each layer index, you can use `pag_applied_layers_index`") from err + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # predict the noise residual + added_cond_kwargs = {"text_embeds": add_text_embeds, "time_ids": add_time_ids} + if ip_adapter_image is not None or ip_adapter_image_embeds is not None: + added_cond_kwargs["image_embeds"] = image_embeds + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + timestep_cond=timestep_cond, + cross_attention_kwargs=self.cross_attention_kwargs, + added_cond_kwargs=added_cond_kwargs, + return_dict=False, + )[0] + + # perform guidance + if self.do_classifier_free_guidance and not self.do_adversarial_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond) + # pag + elif not self.do_classifier_free_guidance and self.do_adversarial_guidance: + noise_pred_original, noise_pred_perturb = noise_pred.chunk(2) + signal_scale = self.pag_scale + if self.do_pag_adaptive_scaling: + signal_scale = self.pag_scale - self.pag_adaptive_scaling * (1000-t) + if signal_scale<0: + signal_scale = 0 + noise_pred = noise_pred_original + signal_scale * (noise_pred_original - noise_pred_perturb) + # both + elif self.do_classifier_free_guidance and self.do_adversarial_guidance: + noise_pred_uncond, noise_pred_text, noise_pred_text_perturb = noise_pred.chunk(3) + signal_scale = self.pag_scale + if self.do_pag_adaptive_scaling: + signal_scale = self.pag_scale - self.pag_adaptive_scaling * (1000-t) + if signal_scale<0: + signal_scale = 0 + noise_pred = noise_pred_text + (self.guidance_scale-1.0) * (noise_pred_text - noise_pred_uncond) + signal_scale * (noise_pred_text - noise_pred_text_perturb) + #+# + + if self.do_classifier_free_guidance and self.guidance_rescale > 0.0: + # Based on 3.4. in https://arxiv.org/pdf/2305.08891.pdf + noise_pred = rescale_noise_cfg(noise_pred, noise_pred_text, guidance_rescale=self.guidance_rescale) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + if XLA_AVAILABLE: + xm.mark_step() + + else: + noise = torch.randn(image.shape, dtype=torch.float16).to(self.unet.device) + guid_latents = self.scheduler.add_noise(image, noise, timesteps[:slider]) + guid_latents = [guid_latents[i:i+1] for i in range(guid_latents.size(0))] + latents = guid_latents[0] + + _b, _c, latent_size_h, latent_size_w=latents.shape + times=torch.ones((1,1,latent_size_h,latent_size_w)).int().to(self.device)*timesteps.max().item() + patch_size = 128 + p=patch_size//2 + + @torch.no_grad() + def create_gradient_border(mask, gradient_width=5): + """ + Needed to average overlapping patches + """ + mask = mask.float().to(self.unet.device) + inverted_mask = mask + distances = F.conv2d(inverted_mask, torch.ones(1, 1, 1, 1, device=device), padding=0) + distance_mask = distances <= gradient_width + kernel_size = gradient_width * 2 + 1 + kernel = torch.ones(1, 1, kernel_size, kernel_size, device=device) / (kernel_size ** 2) + padded_mask = F.pad(inverted_mask, (gradient_width, gradient_width, gradient_width, gradient_width), mode='reflect') + smoothed_distances = F.conv2d(padded_mask, kernel, padding=0).clamp(0, 1) + smoothed_mask = (mask + (1 - mask) * smoothed_distances * distance_mask.float()).clamp(0, 1) + return smoothed_mask + + prev_latents=latents.clone() + + while times.float().mean() >= 0: + + random_indices=self.get_value_coordinates(times[0,0])[0] + i=torch.clamp(random_indices,p,latent_size_h-p).tolist()[0] + j=torch.clamp(random_indices,p,latent_size_w-p).tolist()[1] + + # random patch cropping + sub_latents=self._random_crop(latents, i, j, patch_size) + sub_prev_latents=self._random_crop(prev_latents, i, j, patch_size) + sub_time=self._random_crop(times, i, j, patch_size) + + t = times.max() + ii = torch.where(t==timesteps)[0].item() + + if ii < slider: + sub_guid_latents = self._random_crop(guid_latents[ii], i, j, patch_size) + if ii < len(guid_latents)-1 and ii < slider: + sub_guid_latents_ahead = self._random_crop(guid_latents[ii+1], i, j, patch_size) + + print(f"\r PixelSmith progress: {(1 - times.float().mean() / timesteps.max().item()) * 100:.2f}%",end="") + + if sub_time.float().mean() > 0: + + # Compute the FFT of both sets of latents + fft_sub_latents = torch.fft.rfft2(sub_latents, dim=(-2, -1), norm='ortho') + fft_sub_guid_latents = torch.fft.rfft2(sub_guid_latents, dim=(-2, -1), norm='ortho') + # Calculate magnitude and phase for both FFTs + magnitude_latents = torch.abs(fft_sub_latents) + complex_latents = torch.exp(1j * torch.angle(fft_sub_latents)) + complex_guid_latents = torch.exp(1j * torch.angle(fft_sub_guid_latents)) + # Use the arg function to mix phases + if ii < slider: + mixed_phase = torch.angle(complex_latents + complex_guid_latents) + else: + mixed_phase = torch.angle(fft_sub_latents) + # Reconstruct the complex number using the mixed phase and the original magnitude + fft_sub_latents = magnitude_latents * torch.exp(1j * mixed_phase) + sub_latents = torch.fft.irfft2(fft_sub_latents, dim=(-2, -1), norm='ortho') + + # Generate random numbers for shift directions + shift_left = torch.rand(1).item() < 0.5 + shift_down = torch.rand(1).item() < 0.5 + # + d_rate = 2 + mask_first_row = torch.zeros(1, patch_size) + mask_first_row[:, ::d_rate] = 1 + mask_second_row = torch.roll(mask_first_row, shifts=1, dims=1) + for d in range(1, d_rate): + stacked_rows = torch.concatenate((mask_first_row, mask_second_row), axis=-2) + den_mask = torch.tile(stacked_rows, (patch_size//stacked_rows.shape[0], 1)).to(self.device) + den_mask = den_mask[np.newaxis, np.newaxis, ...].to(self.unet.dtype) + den_mask = torch.roll(den_mask, shifts=(-1 if shift_down else 0, -1 if shift_left else 0), dims=(2, 3)) + + uniques=torch.unique(sub_time) + vmax=uniques[-1] + time_mask=torch.where(sub_time==vmax, 1, 0).to(self.device) + if len(uniques)>1: + sub_latents=sub_latents*time_mask+sub_prev_latents*(time_mask==0) + + #+# + #cfg + if self.do_classifier_free_guidance and not self.do_adversarial_guidance: + latent_model_input = torch.cat([sub_latents] * 2) + #pag + elif not self.do_classifier_free_guidance and self.do_adversarial_guidance: + latent_model_input = torch.cat([sub_latents] * 2) + #both + elif self.do_classifier_free_guidance and self.do_adversarial_guidance: + latent_model_input = torch.cat([sub_latents] * 3) + #no + else: + latent_model_input = sub_latents + + # change attention layer in UNet if use PAG + if self.do_adversarial_guidance: + + if self.do_classifier_free_guidance: + replace_processor = PAGCFGIdentitySelfAttnProcessor() + else: + replace_processor = PAGIdentitySelfAttnProcessor() + if self.pag_applied_layers_index: + drop_layers = self.pag_applied_layers_index + for drop_layer in drop_layers: + layer_number = int(drop_layer[1:]) + try: + if drop_layer[0] == 'd': + down_layers[layer_number].processor = replace_processor + elif drop_layer[0] == 'm': + mid_layers[layer_number].processor = replace_processor + elif drop_layer[0] == 'u': + up_layers[layer_number].processor = replace_processor + else: + raise ValueError(f"Invalid layer type: {drop_layer[0]}") + except IndexError as err: + raise ValueError(f"Invalid layer index: {drop_layer}. Available layers: {len(down_layers)} down layers, {len(mid_layers)} mid layers, {len(up_layers)} up layers.") from err + elif self.pag_applied_layers: + drop_full_layers = self.pag_applied_layers + for drop_full_layer in drop_full_layers: + try: + if drop_full_layer == "down": + for down_layer in down_layers: + down_layer.processor = replace_processor + elif drop_full_layer == "mid": + for mid_layer in mid_layers: + mid_layer.processor = replace_processor + elif drop_full_layer == "up": + for up_layer in up_layers: + up_layer.processor = replace_processor + else: + raise ValueError(f"Invalid layer type: {drop_full_layer}") + except IndexError as err: + raise ValueError(f"Invalid layer index: {drop_full_layer}. Available layers are: down, mid and up. If you need to specify each layer index, you can use `pag_applied_layers_index`") from err + #+# + + latent_model_input = self.scheduler.scale_model_input(latent_model_input, sub_time.max().item()).to(self.unet.dtype) + + add_time_ids[:,0] = latents.shape[-2] * self.vae_scale_factor + add_time_ids[:,4] = latents.shape[-2] * self.vae_scale_factor + add_time_ids[:,1] = latents.shape[-1] * self.vae_scale_factor + add_time_ids[:,5] = latents.shape[-1] * self.vae_scale_factor + add_time_ids[:,2] = (j-64) * self.vae_scale_factor #top + add_time_ids[:,3] = (j-64) * self.vae_scale_factor #left + added_cond_kwargs = {"text_embeds": add_text_embeds, "time_ids": add_time_ids} + if ip_adapter_image is not None or ip_adapter_image_embeds is not None: + added_cond_kwargs["image_embeds"] = image_embeds + noise_pred = self.unet( + latent_model_input, + sub_time.max().item(), + encoder_hidden_states=prompt_embeds, + timestep_cond=timestep_cond, + cross_attention_kwargs=self.cross_attention_kwargs, + added_cond_kwargs=added_cond_kwargs, + return_dict=False, + )[0] + + # perform guidance + if self.do_classifier_free_guidance and not self.do_adversarial_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond) + # pag + elif not self.do_classifier_free_guidance and self.do_adversarial_guidance: + noise_pred_original, noise_pred_perturb = noise_pred.chunk(2) + signal_scale = self.pag_scale + if self.do_pag_adaptive_scaling: + signal_scale = self.pag_scale - self.pag_adaptive_scaling * (1000-sub_time.max().item()) + if signal_scale<0: + signal_scale = 0 + noise_pred = noise_pred_original + signal_scale * (noise_pred_original - noise_pred_perturb) + # both + elif self.do_classifier_free_guidance and self.do_adversarial_guidance: + noise_pred_uncond, noise_pred_text, noise_pred_text_perturb = noise_pred.chunk(3) + signal_scale = self.pag_scale + if self.do_pag_adaptive_scaling: + signal_scale = self.pag_scale - self.pag_adaptive_scaling * (1000-sub_time.max().item()) + if signal_scale<0: + signal_scale = 0 + noise_pred = noise_pred_text + (self.guidance_scale-1.0) * (noise_pred_text - noise_pred_uncond) + signal_scale * (noise_pred_text - noise_pred_text_perturb) + #+# + if self.do_classifier_free_guidance and self.guidance_rescale > 0.0: + # Based on 3.4. in https://arxiv.org/pdf/2305.08891.pdf + noise_pred = rescale_noise_cfg(noise_pred, noise_pred_text, guidance_rescale=self.guidance_rescale) + # compute the previous noisy sample x_t -> x_t-1 + try: + sub_latents = self.scheduler.step(noise_pred, sub_time.max().item(), sub_latents, **extra_step_kwargs, return_dict=False)[0] + except Exception as e: + print('PixelSmith', e) + + smoothed_time_mask = create_gradient_border(time_mask, gradient_width=10) + full_replace_mask = smoothed_time_mask == 1 + no_replace_mask = smoothed_time_mask == 0 + gradient_mask = (smoothed_time_mask > 0) & (smoothed_time_mask < 1) + + if ii(timesteps.min().item()): + next_timestep_index = (timesteps == sub_time.max()).nonzero(as_tuple=True)[0][-1] + next_timestep = timesteps[next_timestep_index + 1].item() + times[...,i-p:i+p,j-p:j+p]=torch.where(sub_time==sub_time.max(), torch.ones_like(sub_time).to(sub_time.device)*next_timestep, sub_time) + else: + times[...,i-p:i+p,j-p:j+p]=torch.where(sub_time==sub_time.max(), torch.ones_like(sub_time).to(sub_time.device)*0, sub_time) + + if torch.all(times == times.max()): + prev_latents=latents.clone() + + if times.float().mean()==0: + break + + if output_type != "latent": + image = self.vae.tiled_decode(latents.to(self.vae.dtype) / self.vae.config.scaling_factor, return_dict=False)[0] + else: + image = latents + + image = self.image_processor.postprocess(image, output_type=output_type) + + self.maybe_free_model_hooks() + + if not return_dict: + return (image,) + + #+# + #Change the attention layers back to original ones after PAG was applied + if self.do_adversarial_guidance: + if self.pag_applied_layers_index: + drop_layers = self.pag_applied_layers_index + for drop_layer in drop_layers: + layer_number = int(drop_layer[1:]) + try: + if drop_layer[0] == 'd': + down_layers[layer_number].processor = AttnProcessor2_0() + elif drop_layer[0] == 'm': + mid_layers[layer_number].processor = AttnProcessor2_0() + elif drop_layer[0] == 'u': + up_layers[layer_number].processor = AttnProcessor2_0() + else: + raise ValueError(f"Invalid layer type: {drop_layer[0]}") + except IndexError as err: + raise ValueError(f"Invalid layer index: {drop_layer}. Available layers: {len(down_layers)} down layers, {len(mid_layers)} mid layers, {len(up_layers)} up layers.") from err + elif self.pag_applied_layers: + drop_full_layers = self.pag_applied_layers + for drop_full_layer in drop_full_layers: + try: + if drop_full_layer == "down": + for down_layer in down_layers: + down_layer.processor = AttnProcessor2_0() + elif drop_full_layer == "mid": + for mid_layer in mid_layers: + mid_layer.processor = AttnProcessor2_0() + elif drop_full_layer == "up": + for up_layer in up_layers: + up_layer.processor = AttnProcessor2_0() + else: + raise ValueError(f"Invalid layer type: {drop_full_layer}") + except IndexError as err: + raise ValueError(f"Invalid layer index: {drop_full_layer}. Available layers are: down, mid and up. If you need to specify each layer index, you can use `pag_applied_layers_index`") from err + #+# + + return ImagePipelineOutput(images=image) + +# %% diff --git a/modules/pixelsmith/vae.py b/modules/pixelsmith/vae.py new file mode 100644 index 000000000..93d376cfd --- /dev/null +++ b/modules/pixelsmith/vae.py @@ -0,0 +1,983 @@ +# Copyright 2023 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from dataclasses import dataclass +from typing import Optional, Tuple + +import numpy as np +import torch +import torch.nn as nn + +from diffusers.utils import BaseOutput, is_torch_version +from diffusers.utils.torch_utils import randn_tensor +from diffusers.models.activations import get_activation +from diffusers.models.attention_processor import SpatialNorm +from diffusers.models.unets.unet_2d_blocks import ( + AutoencoderTinyBlock, + UNetMidBlock2D, + get_down_block, + get_up_block, +) + + +@dataclass +class DecoderOutput(BaseOutput): + r""" + Output of decoding method. + + Args: + sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): + The decoded output sample from the last layer of the model. + """ + + sample: torch.FloatTensor + + +class Encoder(nn.Module): + r""" + The `Encoder` layer of a variational autoencoder that encodes its input into a latent representation. + + Args: + in_channels (`int`, *optional*, defaults to 3): + The number of input channels. + out_channels (`int`, *optional*, defaults to 3): + The number of output channels. + down_block_types (`Tuple[str, ...]`, *optional*, defaults to `("DownEncoderBlock2D",)`): + The types of down blocks to use. See `~diffusers.models.unet_2d_blocks.get_down_block` for available + options. + block_out_channels (`Tuple[int, ...]`, *optional*, defaults to `(64,)`): + The number of output channels for each block. + layers_per_block (`int`, *optional*, defaults to 2): + The number of layers per block. + norm_num_groups (`int`, *optional*, defaults to 32): + The number of groups for normalization. + act_fn (`str`, *optional*, defaults to `"silu"`): + The activation function to use. See `~diffusers.models.activations.get_activation` for available options. + double_z (`bool`, *optional*, defaults to `True`): + Whether to double the number of output channels for the last block. + """ + + def __init__( + self, + in_channels: int = 3, + out_channels: int = 3, + down_block_types: Tuple[str, ...] = ("DownEncoderBlock2D",), + block_out_channels: Tuple[int, ...] = (64,), + layers_per_block: int = 2, + norm_num_groups: int = 32, + act_fn: str = "silu", + double_z: bool = True, + mid_block_add_attention=True, + ): + super().__init__() + self.layers_per_block = layers_per_block + + self.conv_in = nn.Conv2d( + in_channels, + block_out_channels[0], + kernel_size=3, + stride=1, + padding=1, + ) + + self.mid_block = None + self.down_blocks = nn.ModuleList([]) + + # down + output_channel = block_out_channels[0] + for i, down_block_type in enumerate(down_block_types): + input_channel = output_channel + output_channel = block_out_channels[i] + is_final_block = i == len(block_out_channels) - 1 + + down_block = get_down_block( + down_block_type, + num_layers=self.layers_per_block, + in_channels=input_channel, + out_channels=output_channel, + add_downsample=not is_final_block, + resnet_eps=1e-6, + downsample_padding=0, + resnet_act_fn=act_fn, + resnet_groups=norm_num_groups, + attention_head_dim=output_channel, + temb_channels=None, + ) + self.down_blocks.append(down_block) + + # mid + self.mid_block = UNetMidBlock2D( + in_channels=block_out_channels[-1], + resnet_eps=1e-6, + resnet_act_fn=act_fn, + output_scale_factor=1, + resnet_time_scale_shift="default", + attention_head_dim=block_out_channels[-1], + resnet_groups=norm_num_groups, + temb_channels=None, + add_attention=mid_block_add_attention, + ) + + # out + self.conv_norm_out = nn.GroupNorm(num_channels=block_out_channels[-1], num_groups=norm_num_groups, eps=1e-6) + self.conv_act = nn.SiLU() + + conv_out_channels = 2 * out_channels if double_z else out_channels + self.conv_out = nn.Conv2d(block_out_channels[-1], conv_out_channels, 3, padding=1) + + self.gradient_checkpointing = False + + def forward(self, sample: torch.FloatTensor) -> torch.FloatTensor: + r"""The forward method of the `Encoder` class.""" + + sample = self.conv_in(sample) + + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + # down + if is_torch_version(">=", "1.11.0"): + for down_block in self.down_blocks: + sample = torch.utils.checkpoint.checkpoint( + create_custom_forward(down_block), sample, use_reentrant=False + ) + # middle + sample = torch.utils.checkpoint.checkpoint( + create_custom_forward(self.mid_block), sample, use_reentrant=False + ) + else: + for down_block in self.down_blocks: + sample = torch.utils.checkpoint.checkpoint(create_custom_forward(down_block), sample) + # middle + sample = torch.utils.checkpoint.checkpoint(create_custom_forward(self.mid_block), sample) + + else: + # down + for down_block in self.down_blocks: + sample = down_block(sample) + + # middle + sample = self.mid_block(sample) + + # post-process + sample = self.conv_norm_out(sample) + sample = self.conv_act(sample) + sample = self.conv_out(sample) + + return sample + + +class Decoder(nn.Module): + r""" + The `Decoder` layer of a variational autoencoder that decodes its latent representation into an output sample. + + Args: + in_channels (`int`, *optional*, defaults to 3): + The number of input channels. + out_channels (`int`, *optional*, defaults to 3): + The number of output channels. + up_block_types (`Tuple[str, ...]`, *optional*, defaults to `("UpDecoderBlock2D",)`): + The types of up blocks to use. See `~diffusers.models.unet_2d_blocks.get_up_block` for available options. + block_out_channels (`Tuple[int, ...]`, *optional*, defaults to `(64,)`): + The number of output channels for each block. + layers_per_block (`int`, *optional*, defaults to 2): + The number of layers per block. + norm_num_groups (`int`, *optional*, defaults to 32): + The number of groups for normalization. + act_fn (`str`, *optional*, defaults to `"silu"`): + The activation function to use. See `~diffusers.models.activations.get_activation` for available options. + norm_type (`str`, *optional*, defaults to `"group"`): + The normalization type to use. Can be either `"group"` or `"spatial"`. + """ + + def __init__( + self, + in_channels: int = 3, + out_channels: int = 3, + up_block_types: Tuple[str, ...] = ("UpDecoderBlock2D",), + block_out_channels: Tuple[int, ...] = (64,), + layers_per_block: int = 2, + norm_num_groups: int = 32, + act_fn: str = "silu", + norm_type: str = "group", # group, spatial + mid_block_add_attention=True, + ): + super().__init__() + self.layers_per_block = layers_per_block + + self.conv_in = nn.Conv2d( + in_channels, + block_out_channels[-1], + kernel_size=3, + stride=1, + padding=1, + ) + + self.mid_block = None + self.up_blocks = nn.ModuleList([]) + + temb_channels = in_channels if norm_type == "spatial" else None + + # mid + self.mid_block = UNetMidBlock2D( + in_channels=block_out_channels[-1], + resnet_eps=1e-6, + resnet_act_fn=act_fn, + output_scale_factor=1, + resnet_time_scale_shift="default" if norm_type == "group" else norm_type, + attention_head_dim=block_out_channels[-1], + resnet_groups=norm_num_groups, + temb_channels=temb_channels, + add_attention=mid_block_add_attention, + ) + + # up + reversed_block_out_channels = list(reversed(block_out_channels)) + output_channel = reversed_block_out_channels[0] + for i, up_block_type in enumerate(up_block_types): + prev_output_channel = output_channel + output_channel = reversed_block_out_channels[i] + + is_final_block = i == len(block_out_channels) - 1 + + up_block = get_up_block( + up_block_type, + num_layers=self.layers_per_block + 1, + in_channels=prev_output_channel, + out_channels=output_channel, + prev_output_channel=None, + add_upsample=not is_final_block, + resnet_eps=1e-6, + resnet_act_fn=act_fn, + resnet_groups=norm_num_groups, + attention_head_dim=output_channel, + temb_channels=temb_channels, + resnet_time_scale_shift=norm_type, + ) + self.up_blocks.append(up_block) + prev_output_channel = output_channel + + # out + if norm_type == "spatial": + self.conv_norm_out = SpatialNorm(block_out_channels[0], temb_channels) + else: + self.conv_norm_out = nn.GroupNorm(num_channels=block_out_channels[0], num_groups=norm_num_groups, eps=1e-6) + self.conv_act = nn.SiLU() + self.conv_out = nn.Conv2d(block_out_channels[0], out_channels, 3, padding=1) + + self.gradient_checkpointing = False + + def forward( + self, + sample: torch.FloatTensor, + latent_embeds: Optional[torch.FloatTensor] = None, + ) -> torch.FloatTensor: + r"""The forward method of the `Decoder` class.""" + + sample = self.conv_in(sample) + + upscale_dtype = next(iter(self.up_blocks.parameters())).dtype + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + if is_torch_version(">=", "1.11.0"): + # middle + sample = torch.utils.checkpoint.checkpoint( + create_custom_forward(self.mid_block), + sample, + latent_embeds, + use_reentrant=False, + ) + sample = sample.to(upscale_dtype) + + # up + for up_block in self.up_blocks: + sample = torch.utils.checkpoint.checkpoint( + create_custom_forward(up_block), + sample, + latent_embeds, + use_reentrant=False, + ) + else: + # middle + sample = torch.utils.checkpoint.checkpoint( + create_custom_forward(self.mid_block), sample, latent_embeds + ) + sample = sample.to(upscale_dtype) + + # up + for up_block in self.up_blocks: + sample = torch.utils.checkpoint.checkpoint(create_custom_forward(up_block), sample, latent_embeds) + else: + # middle + sample = self.mid_block(sample, latent_embeds) + sample = sample.to(upscale_dtype) + + # up + for up_block in self.up_blocks: + sample = up_block(sample, latent_embeds) + + # post-process + if latent_embeds is None: + sample = self.conv_norm_out(sample) + else: + sample = self.conv_norm_out(sample, latent_embeds) + sample = self.conv_act(sample) + sample = self.conv_out(sample) + + return sample + + +class UpSample(nn.Module): + r""" + The `UpSample` layer of a variational autoencoder that upsamples its input. + + Args: + in_channels (`int`, *optional*, defaults to 3): + The number of input channels. + out_channels (`int`, *optional*, defaults to 3): + The number of output channels. + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + ) -> None: + super().__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.deconv = nn.ConvTranspose2d(in_channels, out_channels, kernel_size=4, stride=2, padding=1) + + def forward(self, x: torch.FloatTensor) -> torch.FloatTensor: + r"""The forward method of the `UpSample` class.""" + x = torch.relu(x) + x = self.deconv(x) + return x + + +class MaskConditionEncoder(nn.Module): + """ + used in AsymmetricAutoencoderKL + """ + + def __init__( + self, + in_ch: int, + out_ch: int = 192, + res_ch: int = 768, + stride: int = 16, + ) -> None: + super().__init__() + + channels = [] + while stride > 1: + stride = stride // 2 + in_ch_ = out_ch * 2 + if out_ch > res_ch: + out_ch = res_ch + if stride == 1: + in_ch_ = res_ch + channels.append((in_ch_, out_ch)) + out_ch *= 2 + + out_channels = [] + for _in_ch, _out_ch in channels: + out_channels.append(_out_ch) + out_channels.append(channels[-1][0]) + + layers = [] + in_ch_ = in_ch + for l in range(len(out_channels)): + out_ch_ = out_channels[l] + if l == 0 or l == 1: + layers.append(nn.Conv2d(in_ch_, out_ch_, kernel_size=3, stride=1, padding=1)) + else: + layers.append(nn.Conv2d(in_ch_, out_ch_, kernel_size=4, stride=2, padding=1)) + in_ch_ = out_ch_ + + self.layers = nn.Sequential(*layers) + + def forward(self, x: torch.FloatTensor, mask=None) -> torch.FloatTensor: + r"""The forward method of the `MaskConditionEncoder` class.""" + out = {} + for l in range(len(self.layers)): + layer = self.layers[l] + x = layer(x) + out[str(tuple(x.shape))] = x + x = torch.relu(x) + return out + + +class MaskConditionDecoder(nn.Module): + r"""The `MaskConditionDecoder` should be used in combination with [`AsymmetricAutoencoderKL`] to enhance the model's + decoder with a conditioner on the mask and masked image. + + Args: + in_channels (`int`, *optional*, defaults to 3): + The number of input channels. + out_channels (`int`, *optional*, defaults to 3): + The number of output channels. + up_block_types (`Tuple[str, ...]`, *optional*, defaults to `("UpDecoderBlock2D",)`): + The types of up blocks to use. See `~diffusers.models.unet_2d_blocks.get_up_block` for available options. + block_out_channels (`Tuple[int, ...]`, *optional*, defaults to `(64,)`): + The number of output channels for each block. + layers_per_block (`int`, *optional*, defaults to 2): + The number of layers per block. + norm_num_groups (`int`, *optional*, defaults to 32): + The number of groups for normalization. + act_fn (`str`, *optional*, defaults to `"silu"`): + The activation function to use. See `~diffusers.models.activations.get_activation` for available options. + norm_type (`str`, *optional*, defaults to `"group"`): + The normalization type to use. Can be either `"group"` or `"spatial"`. + """ + + def __init__( + self, + in_channels: int = 3, + out_channels: int = 3, + up_block_types: Tuple[str, ...] = ("UpDecoderBlock2D",), + block_out_channels: Tuple[int, ...] = (64,), + layers_per_block: int = 2, + norm_num_groups: int = 32, + act_fn: str = "silu", + norm_type: str = "group", # group, spatial + ): + super().__init__() + self.layers_per_block = layers_per_block + + self.conv_in = nn.Conv2d( + in_channels, + block_out_channels[-1], + kernel_size=3, + stride=1, + padding=1, + ) + + self.mid_block = None + self.up_blocks = nn.ModuleList([]) + + temb_channels = in_channels if norm_type == "spatial" else None + + # mid + self.mid_block = UNetMidBlock2D( + in_channels=block_out_channels[-1], + resnet_eps=1e-6, + resnet_act_fn=act_fn, + output_scale_factor=1, + resnet_time_scale_shift="default" if norm_type == "group" else norm_type, + attention_head_dim=block_out_channels[-1], + resnet_groups=norm_num_groups, + temb_channels=temb_channels, + ) + + # up + reversed_block_out_channels = list(reversed(block_out_channels)) + output_channel = reversed_block_out_channels[0] + for i, up_block_type in enumerate(up_block_types): + prev_output_channel = output_channel + output_channel = reversed_block_out_channels[i] + + is_final_block = i == len(block_out_channels) - 1 + + up_block = get_up_block( + up_block_type, + num_layers=self.layers_per_block + 1, + in_channels=prev_output_channel, + out_channels=output_channel, + prev_output_channel=None, + add_upsample=not is_final_block, + resnet_eps=1e-6, + resnet_act_fn=act_fn, + resnet_groups=norm_num_groups, + attention_head_dim=output_channel, + temb_channels=temb_channels, + resnet_time_scale_shift=norm_type, + ) + self.up_blocks.append(up_block) + prev_output_channel = output_channel + + # condition encoder + self.condition_encoder = MaskConditionEncoder( + in_ch=out_channels, + out_ch=block_out_channels[0], + res_ch=block_out_channels[-1], + ) + + # out + if norm_type == "spatial": + self.conv_norm_out = SpatialNorm(block_out_channels[0], temb_channels) + else: + self.conv_norm_out = nn.GroupNorm(num_channels=block_out_channels[0], num_groups=norm_num_groups, eps=1e-6) + self.conv_act = nn.SiLU() + self.conv_out = nn.Conv2d(block_out_channels[0], out_channels, 3, padding=1) + + self.gradient_checkpointing = False + + def forward( + self, + z: torch.FloatTensor, + image: Optional[torch.FloatTensor] = None, + mask: Optional[torch.FloatTensor] = None, + latent_embeds: Optional[torch.FloatTensor] = None, + ) -> torch.FloatTensor: + r"""The forward method of the `MaskConditionDecoder` class.""" + sample = z + sample = self.conv_in(sample) + + upscale_dtype = next(iter(self.up_blocks.parameters())).dtype + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + if is_torch_version(">=", "1.11.0"): + # middle + sample = torch.utils.checkpoint.checkpoint( + create_custom_forward(self.mid_block), + sample, + latent_embeds, + use_reentrant=False, + ) + sample = sample.to(upscale_dtype) + + # condition encoder + if image is not None and mask is not None: + masked_image = (1 - mask) * image + im_x = torch.utils.checkpoint.checkpoint( + create_custom_forward(self.condition_encoder), + masked_image, + mask, + use_reentrant=False, + ) + + # up + for up_block in self.up_blocks: + if image is not None and mask is not None: + sample_ = im_x[str(tuple(sample.shape))] + mask_ = nn.functional.interpolate(mask, size=sample.shape[-2:], mode="nearest") + sample = sample * mask_ + sample_ * (1 - mask_) + sample = torch.utils.checkpoint.checkpoint( + create_custom_forward(up_block), + sample, + latent_embeds, + use_reentrant=False, + ) + if image is not None and mask is not None: + sample = sample * mask + im_x[str(tuple(sample.shape))] * (1 - mask) + else: + # middle + sample = torch.utils.checkpoint.checkpoint( + create_custom_forward(self.mid_block), sample, latent_embeds + ) + sample = sample.to(upscale_dtype) + + # condition encoder + if image is not None and mask is not None: + masked_image = (1 - mask) * image + im_x = torch.utils.checkpoint.checkpoint( + create_custom_forward(self.condition_encoder), + masked_image, + mask, + ) + + # up + for up_block in self.up_blocks: + if image is not None and mask is not None: + sample_ = im_x[str(tuple(sample.shape))] + mask_ = nn.functional.interpolate(mask, size=sample.shape[-2:], mode="nearest") + sample = sample * mask_ + sample_ * (1 - mask_) + sample = torch.utils.checkpoint.checkpoint(create_custom_forward(up_block), sample, latent_embeds) + if image is not None and mask is not None: + sample = sample * mask + im_x[str(tuple(sample.shape))] * (1 - mask) + else: + # middle + sample = self.mid_block(sample, latent_embeds) + sample = sample.to(upscale_dtype) + + # condition encoder + if image is not None and mask is not None: + masked_image = (1 - mask) * image + im_x = self.condition_encoder(masked_image, mask) + + # up + for up_block in self.up_blocks: + if image is not None and mask is not None: + sample_ = im_x[str(tuple(sample.shape))] + mask_ = nn.functional.interpolate(mask, size=sample.shape[-2:], mode="nearest") + sample = sample * mask_ + sample_ * (1 - mask_) + sample = up_block(sample, latent_embeds) + if image is not None and mask is not None: + sample = sample * mask + im_x[str(tuple(sample.shape))] * (1 - mask) + + # post-process + if latent_embeds is None: + sample = self.conv_norm_out(sample) + else: + sample = self.conv_norm_out(sample, latent_embeds) + sample = self.conv_act(sample) + sample = self.conv_out(sample) + + return sample + + +class VectorQuantizer(nn.Module): + """ + Improved version over VectorQuantizer, can be used as a drop-in replacement. Mostly avoids costly matrix + multiplications and allows for post-hoc remapping of indices. + """ + + # NOTE: due to a bug the beta term was applied to the wrong term. for + # backwards compatibility we use the buggy version by default, but you can + # specify legacy=False to fix it. + def __init__( + self, + n_e: int, + vq_embed_dim: int, + beta: float, + remap=None, + unknown_index: str = "random", + sane_index_shape: bool = False, + legacy: bool = True, + ): + super().__init__() + self.n_e = n_e + self.vq_embed_dim = vq_embed_dim + self.beta = beta + self.legacy = legacy + + self.embedding = nn.Embedding(self.n_e, self.vq_embed_dim) + self.embedding.weight.data.uniform_(-1.0 / self.n_e, 1.0 / self.n_e) + + self.remap = remap + if self.remap is not None: + self.register_buffer("used", torch.tensor(np.load(self.remap))) + self.used: torch.Tensor + self.re_embed = self.used.shape[0] + self.unknown_index = unknown_index # "random" or "extra" or integer + if self.unknown_index == "extra": + self.unknown_index = self.re_embed + self.re_embed = self.re_embed + 1 + print( + f"Remapping {self.n_e} indices to {self.re_embed} indices. " + f"Using {self.unknown_index} for unknown indices." + ) + else: + self.re_embed = n_e + + self.sane_index_shape = sane_index_shape + + def remap_to_used(self, inds: torch.LongTensor) -> torch.LongTensor: + ishape = inds.shape + assert len(ishape) > 1 + inds = inds.reshape(ishape[0], -1) + used = self.used.to(inds) + match = (inds[:, :, None] == used[None, None, ...]).long() + new = match.argmax(-1) + unknown = match.sum(2) < 1 + if self.unknown_index == "random": + new[unknown] = torch.randint(0, self.re_embed, size=new[unknown].shape).to(device=new.device) + else: + new[unknown] = self.unknown_index + return new.reshape(ishape) + + def unmap_to_all(self, inds: torch.LongTensor) -> torch.LongTensor: + ishape = inds.shape + assert len(ishape) > 1 + inds = inds.reshape(ishape[0], -1) + used = self.used.to(inds) + if self.re_embed > self.used.shape[0]: # extra token + inds[inds >= self.used.shape[0]] = 0 # simply set to zero + back = torch.gather(used[None, :][inds.shape[0] * [0], :], 1, inds) + return back.reshape(ishape) + + def forward(self, z: torch.FloatTensor) -> Tuple[torch.FloatTensor, torch.FloatTensor, Tuple]: + # reshape z -> (batch, height, width, channel) and flatten + z = z.permute(0, 2, 3, 1).contiguous() + z_flattened = z.view(-1, self.vq_embed_dim) + + # distances from z to embeddings e_j (z - e)^2 = z^2 + e^2 - 2 e * z + min_encoding_indices = torch.argmin(torch.cdist(z_flattened, self.embedding.weight), dim=1) + + z_q = self.embedding(min_encoding_indices).view(z.shape) + perplexity = None + min_encodings = None + + # compute loss for embedding + if not self.legacy: + loss = self.beta * torch.mean((z_q.detach() - z) ** 2) + torch.mean((z_q - z.detach()) ** 2) + else: + loss = torch.mean((z_q.detach() - z) ** 2) + self.beta * torch.mean((z_q - z.detach()) ** 2) + + # preserve gradients + z_q: torch.FloatTensor = z + (z_q - z).detach() + + # reshape back to match original input shape + z_q = z_q.permute(0, 3, 1, 2).contiguous() + + if self.remap is not None: + min_encoding_indices = min_encoding_indices.reshape(z.shape[0], -1) # add batch axis + min_encoding_indices = self.remap_to_used(min_encoding_indices) + min_encoding_indices = min_encoding_indices.reshape(-1, 1) # flatten + + if self.sane_index_shape: + min_encoding_indices = min_encoding_indices.reshape(z_q.shape[0], z_q.shape[2], z_q.shape[3]) + + return z_q, loss, (perplexity, min_encodings, min_encoding_indices) + + def get_codebook_entry(self, indices: torch.LongTensor, shape: Tuple[int, ...]) -> torch.FloatTensor: + # shape specifying (batch, height, width, channel) + if self.remap is not None: + indices = indices.reshape(shape[0], -1) # add batch axis + indices = self.unmap_to_all(indices) + indices = indices.reshape(-1) # flatten again + + # get quantized latent vectors + z_q: torch.FloatTensor = self.embedding(indices) + + if shape is not None: + z_q = z_q.view(shape) + # reshape back to match original input shape + z_q = z_q.permute(0, 3, 1, 2).contiguous() + + return z_q + + +class DiagonalGaussianDistribution(object): + def __init__(self, parameters: torch.Tensor, deterministic: bool = False): + self.parameters = parameters + self.mean, self.logvar = torch.chunk(parameters, 2, dim=1) + self.logvar = torch.clamp(self.logvar, -30.0, 20.0) + self.deterministic = deterministic + self.std = torch.exp(0.5 * self.logvar) + self.var = torch.exp(self.logvar) + if self.deterministic: + self.var = self.std = torch.zeros_like( + self.mean, device=self.parameters.device, dtype=self.parameters.dtype + ) + + def sample(self, generator: Optional[torch.Generator] = None) -> torch.FloatTensor: + # make sure sample is on the same device as the parameters and has same dtype + sample = randn_tensor( + self.mean.shape, + generator=generator, + device=self.parameters.device, + dtype=self.parameters.dtype, + ) + x = self.mean + self.std * sample + return x + + def kl(self, other: "DiagonalGaussianDistribution" = None) -> torch.Tensor: + if self.deterministic: + return torch.Tensor([0.0]) + else: + if other is None: + return 0.5 * torch.sum( + torch.pow(self.mean, 2) + self.var - 1.0 - self.logvar, + dim=[1, 2, 3], + ) + else: + return 0.5 * torch.sum( + torch.pow(self.mean - other.mean, 2) / other.var + + self.var / other.var + - 1.0 + - self.logvar + + other.logvar, + dim=[1, 2, 3], + ) + + def nll(self, sample: torch.Tensor, dims: Tuple[int, ...] = [1, 2, 3]) -> torch.Tensor: + if self.deterministic: + return torch.Tensor([0.0]) + logtwopi = np.log(2.0 * np.pi) + return 0.5 * torch.sum( + logtwopi + self.logvar + torch.pow(sample - self.mean, 2) / self.var, + dim=dims, + ) + + def mode(self) -> torch.Tensor: + return self.mean + + +class EncoderTiny(nn.Module): + r""" + The `EncoderTiny` layer is a simpler version of the `Encoder` layer. + + Args: + in_channels (`int`): + The number of input channels. + out_channels (`int`): + The number of output channels. + num_blocks (`Tuple[int, ...]`): + Each value of the tuple represents a Conv2d layer followed by `value` number of `AutoencoderTinyBlock`'s to + use. + block_out_channels (`Tuple[int, ...]`): + The number of output channels for each block. + act_fn (`str`): + The activation function to use. See `~diffusers.models.activations.get_activation` for available options. + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + num_blocks: Tuple[int, ...], + block_out_channels: Tuple[int, ...], + act_fn: str, + ): + super().__init__() + + layers = [] + for i, num_block in enumerate(num_blocks): + num_channels = block_out_channels[i] + + if i == 0: + layers.append(nn.Conv2d(in_channels, num_channels, kernel_size=3, padding=1)) + else: + layers.append( + nn.Conv2d( + num_channels, + num_channels, + kernel_size=3, + padding=1, + stride=2, + bias=False, + ) + ) + + for _ in range(num_block): + layers.append(AutoencoderTinyBlock(num_channels, num_channels, act_fn)) + + layers.append(nn.Conv2d(block_out_channels[-1], out_channels, kernel_size=3, padding=1)) + + self.layers = nn.Sequential(*layers) + self.gradient_checkpointing = False + + def forward(self, x: torch.FloatTensor) -> torch.FloatTensor: + r"""The forward method of the `EncoderTiny` class.""" + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + if is_torch_version(">=", "1.11.0"): + x = torch.utils.checkpoint.checkpoint(create_custom_forward(self.layers), x, use_reentrant=False) + else: + x = torch.utils.checkpoint.checkpoint(create_custom_forward(self.layers), x) + + else: + # scale image from [-1, 1] to [0, 1] to match TAESD convention + x = self.layers(x.add(1).div(2)) + + return x + + +class DecoderTiny(nn.Module): + r""" + The `DecoderTiny` layer is a simpler version of the `Decoder` layer. + + Args: + in_channels (`int`): + The number of input channels. + out_channels (`int`): + The number of output channels. + num_blocks (`Tuple[int, ...]`): + Each value of the tuple represents a Conv2d layer followed by `value` number of `AutoencoderTinyBlock`'s to + use. + block_out_channels (`Tuple[int, ...]`): + The number of output channels for each block. + upsampling_scaling_factor (`int`): + The scaling factor to use for upsampling. + act_fn (`str`): + The activation function to use. See `~diffusers.models.activations.get_activation` for available options. + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + num_blocks: Tuple[int, ...], + block_out_channels: Tuple[int, ...], + upsampling_scaling_factor: int, + act_fn: str, + ): + super().__init__() + + layers = [ + nn.Conv2d(in_channels, block_out_channels[0], kernel_size=3, padding=1), + get_activation(act_fn), + ] + + for i, num_block in enumerate(num_blocks): + is_final_block = i == (len(num_blocks) - 1) + num_channels = block_out_channels[i] + + for _ in range(num_block): + layers.append(AutoencoderTinyBlock(num_channels, num_channels, act_fn)) + + if not is_final_block: + layers.append(nn.Upsample(scale_factor=upsampling_scaling_factor)) + + conv_out_channel = num_channels if not is_final_block else out_channels + layers.append( + nn.Conv2d( + num_channels, + conv_out_channel, + kernel_size=3, + padding=1, + bias=is_final_block, + ) + ) + + self.layers = nn.Sequential(*layers) + self.gradient_checkpointing = False + + def forward(self, x: torch.FloatTensor) -> torch.FloatTensor: + r"""The forward method of the `DecoderTiny` class.""" + # Clamp. + x = torch.tanh(x / 3) * 3 + + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + if is_torch_version(">=", "1.11.0"): + x = torch.utils.checkpoint.checkpoint(create_custom_forward(self.layers), x, use_reentrant=False) + else: + x = torch.utils.checkpoint.checkpoint(create_custom_forward(self.layers), x) + + else: + x = self.layers(x) + + # scale image from [0, 1] to [-1, 1] to match diffusers convention + return x.mul(2).sub(1) diff --git a/modules/sd_models.py b/modules/sd_models.py index 51d25edfa..d9e07e1de 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -1208,6 +1208,7 @@ def set_diffuser_pipe(pipe, new_pipe_type): 'FluxFillPipeline', 'FluxControlPipeline', 'StableVideoDiffusionPipeline', + 'PixelSmithXLPipeline', ] has_errors = False diff --git a/scripts/pixelsmith.py b/scripts/pixelsmith.py new file mode 100644 index 000000000..a9509081c --- /dev/null +++ b/scripts/pixelsmith.py @@ -0,0 +1,78 @@ +import gradio as gr +from PIL import Image +from modules import scripts, processing, shared, sd_models, devices, images + + +class Script(scripts.Script): + def __init__(self): + super().__init__() + self.orig_pipe = None + self.orig_vae = None + self.vae = None + + def title(self): + return 'PixelSmith' + + def show(self, is_img2img): + return shared.native + + def ui(self, _is_img2img): # ui elements + with gr.Row(): + gr.HTML('  PixelSmith
') + with gr.Row(): + slider = gr.Slider(label="Slider", value=20, minimum=0, maximum=100, step=1) + return [slider] + + def encode(self, p: processing.StableDiffusionProcessing, image: Image.Image): + if image is None: + return None + import numpy as np + import torch + if p.width is None or p.width == 0: + p.width = int(8 * (image.width * p.scale_by // 8)) + if p.height is None or p.height == 0: + p.height = int(8 * (image.height * p.scale_by // 8)) + image = images.resize_image(p.resize_mode, image, p.width, p.height, upscaler_name=p.resize_name, context=p.resize_context) + tensor = np.array(image).astype(np.float16) / 255.0 + tensor = tensor[None].transpose(0, 3, 1, 2) + # image = image.transpose(0, 3, 1, 2) + tensor = torch.from_numpy(tensor).to(device=devices.device, dtype=devices.dtype) + tensor = 2.0 * tensor - 1.0 + with devices.inference_context(): + latent = shared.sd_model.vae.tiled_encode(tensor) + latent = shared.sd_model.vae.config.scaling_factor * latent.latent_dist.sample() + shared.log.info(f'PixelSmith encode: image={image} latent={latent.shape} width={p.width} height={p.height} vae={shared.sd_model.vae.__class__.__name__}') + return latent + + + def run(self, p: processing.StableDiffusionProcessing, slider: int = 20): # pylint: disable=arguments-differ + supported_model_list = ['sdxl'] + if shared.sd_model_type not in supported_model_list: + shared.log.warning(f'PixelSmith: class={shared.sd_model.__class__.__name__} model={shared.sd_model_type} required={supported_model_list}') + from modules.pixelsmith import PixelSmithXLPipeline, PixelSmithVAE + self.orig_pipe = shared.sd_model + self.orig_vae = shared.sd_model.vae + if self.vae is None: + self.vae = PixelSmithVAE.from_pretrained("madebyollin/sdxl-vae-fp16-fix", torch_dtype=devices.dtype).to(devices.device) + shared.sd_model = sd_models.switch_pipe(PixelSmithXLPipeline, shared.sd_model) + shared.sd_model.vae = self.vae + shared.sd_model.vae.enable_tiling() + p.extra_generation_params["PixelSmith"] = f'Slider={slider}' + p.sampler_name = 'DDIM' + p.task_args['slider'] = slider + # p.task_args['output_type'] = 'pil' + if hasattr(p, 'init_images') and p.init_images is not None and len(p.init_images) > 0: + p.task_args['image'] = self.encode(p, p.init_images[0]) + p.init_images = None + shared.log.info(f'PixelSmith apply: slider={slider} class={shared.sd_model.__class__.__name__} vae={shared.sd_model.vae.__class__.__name__}') + # processed = processing.process_images(p) + + def after(self, p: processing.StableDiffusionProcessing, processed: processing.Processed, slider): # pylint: disable=unused-argument + if self.orig_pipe is None: + return processed + if shared.sd_model.__class__.__name__ == 'PixelSmithXLPipeline': + shared.sd_model = self.orig_pipe + shared.sd_model.vae = self.orig_vae + self.orig_pipe = None + self.orig_vae = None + return processed From 17aaa6f33ce7a0991fc59738ce51fcfc1c046cd8 Mon Sep 17 00:00:00 2001 From: Seunghoon Lee Date: Sat, 4 Jan 2025 02:21:51 +0900 Subject: [PATCH 217/249] zluda experimental hipblaslt support --- modules/zluda_hijacks.py | 6 ++++-- modules/zluda_installer.py | 33 ++++++++++++++++++++++++++++++--- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/modules/zluda_hijacks.py b/modules/zluda_hijacks.py index 0f42a5448..cd41bcb72 100644 --- a/modules/zluda_hijacks.py +++ b/modules/zluda_hijacks.py @@ -1,5 +1,5 @@ import torch -from modules import rocm +from modules import zluda_installer, rocm _topk = torch.topk @@ -35,4 +35,6 @@ def do_hijack(): torch.fft.fftn = fft_fftn torch.fft.ifftn = fft_ifftn torch.fft.rfftn = fft_rfftn - torch.jit.script = jit_script + + if not zluda_installer.experimental_hipBLASLt_support: + torch.jit.script = jit_script diff --git a/modules/zluda_installer.py b/modules/zluda_installer.py index 5e43a6635..89dac35f2 100644 --- a/modules/zluda_installer.py +++ b/modules/zluda_installer.py @@ -1,8 +1,10 @@ import os import sys +import site import ctypes import shutil import zipfile +import subprocess import urllib.request from typing import Optional, Union from modules import rocm @@ -13,8 +15,10 @@ 'cusparse.dll': 'cusparse64_11.dll', 'nvrtc.dll': 'nvrtc64_112_0.dll', } -HIPSDK_TARGETS = ['rocblas.dll', 'rocsolver.dll', f'hiprtc{"".join([v.zfill(2) for v in rocm.version.split(".")])}.dll'] +HIPSDK_TARGETS = ['rocblas.dll', 'rocsolver.dll'] ZLUDA_TARGETS = ('nvcuda.dll', 'nvml.dll',) +version: Union[str, None] = None +experimental_hipBLASLt_support: bool = False default_agent: Union[rocm.Agent, None] = None @@ -28,7 +32,9 @@ def set_default_agent(agent: rocm.Agent): def install(zluda_path: os.PathLike) -> None: + if os.path.exists(zluda_path): + __initialize(zluda_path) return commit = os.environ.get("ZLUDA_HASH", "1b6e012d8f2404840b524e2abae12cb91e1ac01d") @@ -42,6 +48,7 @@ def install(zluda_path: os.PathLike) -> None: info.filename = os.path.basename(info.filename) archive.extract(info, '.zluda') os.remove('_zluda') + __initialize(zluda_path) def uninstall() -> None: @@ -50,6 +57,8 @@ def uninstall() -> None: def make_copy(zluda_path: os.PathLike) -> None: + __initialize(zluda_path) + for k, v in DLL_MAPPING.items(): if not os.path.exists(os.path.join(zluda_path, v)): try: @@ -60,6 +69,7 @@ def make_copy(zluda_path: os.PathLike) -> None: def load(zluda_path: os.PathLike) -> None: os.environ["ZLUDA_COMGR_LOG_LEVEL"] = "1" + os.environ["ZLUDA_NVRTC_LIB"] = os.path.join([v for v in site.getsitepackages() if v.endswith("site-packages")][0], "torch", "lib", "nvrtc64_112_0.dll") for v in HIPSDK_TARGETS: ctypes.windll.LoadLibrary(os.path.join(rocm.path, 'bin', v)) @@ -86,7 +96,24 @@ def _join_rocm_home(*paths) -> str: def get_default_torch_version(agent: Optional[rocm.Agent]) -> str: if agent is not None: if agent.arch in (rocm.MicroArchitecture.RDNA, rocm.MicroArchitecture.CDNA,): - return "2.3.1" + return "2.4.1" if experimental_hipBLASLt_support else "2.3.1" elif agent.arch == rocm.MicroArchitecture.GCN: return "2.2.1" - return "2.3.1" + return "2.4.1" if experimental_hipBLASLt_support else "2.3.1" + + +def __initialize(zluda_path: os.PathLike): + global version, experimental_hipBLASLt_support # pylint: disable=global-statement + + if version is not None: + return + + process = subprocess.run(["zluda", "--version"], cwd=zluda_path, shell=True, check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + version = process.stdout.decode(encoding="utf8", errors="ignore")[6:].strip() + experimental_hipBLASLt_support = version == "3.8.5.pre1" + + if experimental_hipBLASLt_support: + HIPSDK_TARGETS.append('hipblaslt.dll') + DLL_MAPPING['cublasLt.dll'] = 'cublasLt64_11.dll' + else: + HIPSDK_TARGETS.append(f'hiprtc{"".join([v.zfill(2) for v in rocm.version.split(".")])}.dll') From 14d462ba520ac6a01dd876b1c65a07dc2c982add Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sat, 4 Jan 2025 13:55:57 -0500 Subject: [PATCH 218/249] improve metadata Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 4 +++- modules/processing_args.py | 3 ++- modules/processing_info.py | 26 ++++++++++++++------------ modules/ui_control.py | 6 ++++-- modules/ui_img2img.py | 27 ++++++++++++++++++++++++--- modules/ui_txt2img.py | 12 ++++++------ wiki | 2 +- 7 files changed, 54 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f2bd7d4b7..0be26a812 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Change Log for SD.Next -## Update for 2025-01-03 +## Update for 2025-01-04 - [Allegro Video](https://huggingface.co/rhymes-ai/Allegro) - optimizations: full offload and quantization support @@ -20,6 +20,8 @@ - add explicit detailer steps setting - **SysInfo**: - update to collected data and benchmarks +- **Metadata**: + - improved metadata save and restore - **Fixes**: - explict clear caches on model load - lock adetailer commit: `#a89c01d` diff --git a/modules/processing_args.py b/modules/processing_args.py index 360928de7..32fac0b17 100644 --- a/modules/processing_args.py +++ b/modules/processing_args.py @@ -35,7 +35,8 @@ def task_specific_kwargs(p, model): elif (sd_models.get_diffusers_task(model) == sd_models.DiffusersTaskType.IMAGE_2_IMAGE or is_img2img_model) and len(getattr(p, 'init_images', [])) > 0: if shared.sd_model_type == 'sdxl' and hasattr(model, 'register_to_config'): model.register_to_config(requires_aesthetics_score = False) - p.ops.append('img2img') + if 'hires' not in p.ops: + p.ops.append('img2img') task_args = { 'image': p.init_images, 'strength': p.denoising_strength, diff --git a/modules/processing_info.py b/modules/processing_info.py index e0fca12ae..54bfe331d 100644 --- a/modules/processing_info.py +++ b/modules/processing_info.py @@ -37,7 +37,6 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts=None, all_seeds=No all_negative_prompts.append(all_negative_prompts[-1]) comment = ', '.join(comments) if comments is not None and type(comments) is list else None ops = list(set(p.ops)) - ops.reverse() args = { # basic "Size": f"{p.width}x{p.height}" if hasattr(p, 'width') and hasattr(p, 'height') else None, @@ -46,6 +45,7 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts=None, all_seeds=No "Seed": all_seeds[index], "Seed resize from": None if p.seed_resize_from_w == 0 or p.seed_resize_from_h == 0 else f"{p.seed_resize_from_w}x{p.seed_resize_from_h}", "CFG scale": p.cfg_scale if p.cfg_scale > 1.0 else None, + "CFG rescale": p.diffusers_guidance_rescale if p.diffusers_guidance_rescale > 0 else None, "CFG end": p.cfg_end if p.cfg_end < 1.0 else None, "Clip skip": p.clip_skip if p.clip_skip > 1 else None, "Batch": f'{p.n_iter}x{p.batch_size}' if p.n_iter > 1 or p.batch_size > 1 else None, @@ -82,31 +82,33 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts=None, all_seeds=No args["Variation strength"] = p.subseed_strength if p.subseed_strength > 0 else None if 'hires' in p.ops or 'upscale' in p.ops: is_resize = p.hr_resize_mode > 0 and (p.hr_upscaler != 'None' or p.hr_resize_mode == 5) + is_fixed = p.hr_resize_x > 0 or p.hr_resize_y > 0 args["Refine"] = p.enable_hr args["Hires force"] = p.hr_force args["Hires steps"] = p.hr_second_pass_steps - args["HiRes resize mode"] = p.hr_resize_mode if is_resize else None - args["HiRes resize context"] = p.hr_resize_context if p.hr_resize_mode == 5 else None + args["HiRes mode"] = p.hr_resize_mode if is_resize else None + args["HiRes context"] = p.hr_resize_context if p.hr_resize_mode == 5 else None args["Hires upscaler"] = p.hr_upscaler if is_resize else None - args["Hires scale"] = p.hr_scale if is_resize else None - args["Hires resize"] = f"{p.hr_resize_x}x{p.hr_resize_y}" if is_resize else None + if is_fixed: + args["Hires fixed"] = f"{p.hr_resize_x}x{p.hr_resize_y}" if is_resize else None + else: + args["Hires scale"] = p.hr_scale if is_resize else None args["Hires size"] = f"{p.hr_upscale_to_x}x{p.hr_upscale_to_y}" if is_resize else None - args["Denoising strength"] = p.denoising_strength - args["Hires sampler"] = p.hr_sampler_name - args["Image CFG scale"] = p.image_cfg_scale - args["CFG rescale"] = p.diffusers_guidance_rescale + args["Hires strength"] = p.denoising_strength + args["Hires sampler"] = p.hr_sampler_name if p.hr_sampler_name != p.sampler_name else None + args["Hires CFG scale"] = p.image_cfg_scale if 'refine' in p.ops: args["Refine"] = p.enable_hr args["Refiner"] = None if (not shared.opts.add_model_name_to_info) or (not shared.sd_refiner) or (not shared.sd_refiner.sd_checkpoint_info.model_name) else shared.sd_refiner.sd_checkpoint_info.model_name.replace(',', '').replace(':', '') - args['Image CFG scale'] = p.image_cfg_scale + args['Hires CFG scale'] = p.image_cfg_scale args['Refiner steps'] = p.refiner_steps args['Refiner start'] = p.refiner_start args["Hires steps"] = p.hr_second_pass_steps args["Hires sampler"] = p.hr_sampler_name - args["CFG rescale"] = p.diffusers_guidance_rescale - if 'img2img' in p.ops or 'inpaint' in p.ops: + if ('img2img' in p.ops or 'inpaint' in p.ops) and ('txt2img' not in p.ops and 'hires' not in p.ops): # real img2img/inpaint args["Init image size"] = f"{getattr(p, 'init_img_width', 0)}x{getattr(p, 'init_img_height', 0)}" args["Init image hash"] = getattr(p, 'init_img_hash', None) + args['Image CFG scale'] = p.image_cfg_scale args['Resize scale'] = getattr(p, 'scale_by', None) args["Mask weight"] = getattr(p, "inpainting_mask_weight", shared.opts.inpainting_mask_weight) if p.is_using_inpainting_conditioning else None args["Denoising strength"] = getattr(p, 'denoising_strength', None) diff --git a/modules/ui_control.py b/modules/ui_control.py index c12a03130..4dbd1b089 100644 --- a/modules/ui_control.py +++ b/modules/ui_control.py @@ -649,6 +649,7 @@ def create_ui(_blocks: gr.Blocks=None): (cfg_end, "CFG end"), (clip_skip, "Clip skip"), (image_cfg_scale, "Image CFG scale"), + (image_cfg_scale, "Hires CFG scale"), (guidance_rescale, "CFG rescale"), (full_quality, "Full quality"), (detailer, "Detailer"), @@ -658,13 +659,14 @@ def create_ui(_blocks: gr.Blocks=None): (enable_hr, "Second pass"), (enable_hr, "Refine"), (hr_sampler_index, "Hires sampler"), + (denoising_strength, "Hires strength"), (denoising_strength, "Denoising strength"), (hr_upscaler, "Hires upscaler"), (hr_force, "Hires force"), (hr_second_pass_steps, "Hires steps"), (hr_scale, "Hires upscale"), - (hr_resize_x, "Hires resize-1"), - (hr_resize_y, "Hires resize-2"), + (hr_resize_x, "Hires fixed-1"), + (hr_resize_y, "Hires fixed-2"), # refiner (refiner_start, "Refiner start"), (refiner_steps, "Refiner steps"), diff --git a/modules/ui_img2img.py b/modules/ui_img2img.py index 45c901c6d..48f22af7e 100644 --- a/modules/ui_img2img.py +++ b/modules/ui_img2img.py @@ -253,19 +253,40 @@ def select_img2img_tab(tab): (seed, "Seed"), (subseed, "Variation seed"), (subseed_strength, "Variation strength"), - # denoise - (denoising_strength, "Denoising strength"), - (refiner_start, "Refiner start"), # advanced (cfg_scale, "CFG scale"), (cfg_end, "CFG end"), (image_cfg_scale, "Image CFG scale"), + (image_cfg_scale, "Hires CFG scale"), (clip_skip, "Clip skip"), (diffusers_guidance_rescale, "CFG rescale"), (full_quality, "Full quality"), (detailer, "Detailer"), (tiling, "Tiling"), (hidiffusion, "HiDiffusion"), + # second pass + (enable_hr, "Second pass"), + (enable_hr, "Refine"), + (denoising_strength, "Denoising strength"), + (denoising_strength, "Hires strength"), + (hr_sampler_index, "Hires sampler"), + (hr_resize_mode, "Hires mode"), + (hr_resize_context, "Hires context"), + (hr_upscaler, "Hires upscaler"), + (hr_force, "Hires force"), + (hr_second_pass_steps, "Hires steps"), + (hr_scale, "Hires upscale"), + (hr_scale, "Hires scale"), + (hr_resize_x, "Hires fixed-1"), + (hr_resize_y, "Hires fixed-2"), + # refiner + (refiner_start, "Refiner start"), + (refiner_steps, "Refiner steps"), + (refiner_prompt, "Prompt2"), + (refiner_negative, "Negative2"), + # pag + (pag_scale, "PAG scale"), + (pag_adaptive, "PAG adaptive"), # inpaint (mask_blur, "Mask blur"), (mask_alpha, "Mask alpha"), diff --git a/modules/ui_txt2img.py b/modules/ui_txt2img.py index e2886e901..be27b7e9e 100644 --- a/modules/ui_txt2img.py +++ b/modules/ui_txt2img.py @@ -119,6 +119,7 @@ def create_ui(): (cfg_end, "CFG end"), (clip_skip, "Clip skip"), (image_cfg_scale, "Image CFG scale"), + (image_cfg_scale, "Hires CFG scale"), (diffusers_guidance_rescale, "CFG rescale"), (full_quality, "Full quality"), (detailer, "Detailer"), @@ -128,18 +129,17 @@ def create_ui(): (enable_hr, "Second pass"), (enable_hr, "Refine"), (denoising_strength, "Denoising strength"), + (denoising_strength, "Hires strength"), (hr_sampler_index, "Hires sampler"), - (hr_resize_mode, "Hires resize mode"), - (hr_resize_context, "Hires resize context"), + (hr_resize_mode, "Hires mode"), + (hr_resize_context, "Hires context"), (hr_upscaler, "Hires upscaler"), (hr_force, "Hires force"), (hr_second_pass_steps, "Hires steps"), (hr_scale, "Hires upscale"), (hr_scale, "Hires scale"), - (hr_resize_x, "Hires resize-1"), - (hr_resize_y, "Hires resize-2"), - (hr_resize_x, "Hires size-1"), - (hr_resize_y, "Hires size-2"), + (hr_resize_x, "Hires fixed-1"), + (hr_resize_y, "Hires fixed-2"), # refiner (refiner_start, "Refiner start"), (refiner_steps, "Refiner steps"), diff --git a/wiki b/wiki index 7bd8f8200..1f792d308 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 7bd8f82008007bc7a766ca47b4dc1a54470397df +Subproject commit 1f792d30858b511ef728531250f3c1a283e4c211 From a1fe0f4d6dcc62ad1d545ce5b72c827762998030 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 5 Jan 2025 08:13:41 -0500 Subject: [PATCH 219/249] fix pulid Signed-off-by: Vladimir Mandic --- modules/processing_args.py | 4 +++- modules/processing_callbacks.py | 13 +++++++++---- modules/progress.py | 2 +- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/modules/processing_args.py b/modules/processing_args.py index 32fac0b17..529971d5c 100644 --- a/modules/processing_args.py +++ b/modules/processing_args.py @@ -109,8 +109,10 @@ def set_pipeline_args(p, model, prompts:list, negative_prompts:list, prompts_2:t if hasattr(model, "set_progress_bar_config"): model.set_progress_bar_config(bar_format='Progress {rate_fmt}{postfix} {bar} {percentage:3.0f}% {n_fmt}/{total_fmt} {elapsed} {remaining} ' + '\x1b[38;5;71m' + desc, ncols=80, colour='#327fba') args = {} + has_vae = hasattr(model, 'vae') or (hasattr(model, 'pipe') and hasattr(model.pipe, 'vae')) if hasattr(model, 'pipe') and not hasattr(model, 'no_recurse'): # recurse model = model.pipe + has_vae = has_vae or hasattr(model, 'vae') signature = inspect.signature(type(model).__call__, follow_wrapped=True) possible = list(signature.parameters) @@ -233,7 +235,7 @@ def set_pipeline_args(p, model, prompts:list, negative_prompts:list, prompts_2:t if sd_models.get_diffusers_task(model) == sd_models.DiffusersTaskType.TEXT_2_IMAGE: args['latents'] = p.init_latent if 'output_type' in possible: - if not hasattr(model, 'vae'): + if not has_vae: kwargs['output_type'] = 'np' # only set latent if model has vae # model specific diff --git a/modules/processing_callbacks.py b/modules/processing_callbacks.py index a9c6fbcf7..d3e57818b 100644 --- a/modules/processing_callbacks.py +++ b/modules/processing_callbacks.py @@ -27,7 +27,7 @@ def prompt_callback(step, kwargs): assert prompt_embeds.shape == kwargs['prompt_embeds'].shape, f"prompt_embed shape mismatch {kwargs['prompt_embeds'].shape} {prompt_embeds.shape}" kwargs['prompt_embeds'] = prompt_embeds except Exception as e: - debug_callback(f"Callback: {e}") + debug_callback(f"Callback: type=prompt {e}") return kwargs @@ -115,11 +115,16 @@ def diffusers_callback(pipe, step: int = 0, timestep: int = 0, kwargs: dict = {} shared.state.current_latent = kwargs['latents'] shared.state.current_noise_pred = current_noise_pred - if hasattr(pipe, "scheduler") and hasattr(pipe.scheduler, "sigmas") and hasattr(pipe.scheduler, "step_index"): - shared.state.current_sigma = pipe.scheduler.sigmas[pipe.scheduler.step_index - 1] - shared.state.current_sigma_next = pipe.scheduler.sigmas[pipe.scheduler.step_index] + if hasattr(pipe, "scheduler") and hasattr(pipe.scheduler, "sigmas") and hasattr(pipe.scheduler, "step_index") and pipe.scheduler.step_index is not None: + try: + shared.state.current_sigma = pipe.scheduler.sigmas[pipe.scheduler.step_index-1] + shared.state.current_sigma_next = pipe.scheduler.sigmas[pipe.scheduler.step_index] + except Exception: + pass except Exception as e: shared.log.error(f'Callback: {e}') + # from modules import errors + # errors.display(e, 'Callback') if shared.cmd_opts.profile and shared.profiler is not None: shared.profiler.step() t1 = time.time() diff --git a/modules/progress.py b/modules/progress.py index bc3e5500c..f6b47f1c0 100644 --- a/modules/progress.py +++ b/modules/progress.py @@ -80,7 +80,7 @@ def progressapi(req: ProgressRequest): id_live_preview = req.id_live_preview live_preview = None updated = shared.state.set_current_image() - debug_log(f'Preview: job={shared.state.job} active={active} progress={current}/{total} step={shared.state.current_image_sampling_step}/{shared.state.sampling_step} request={id_live_preview} last={shared.state.id_live_preview} enabled={shared.opts.live_previews_enable} job={shared.state.preview_job} updated={updated} image={shared.state.current_image} elapsed={elapsed:.3f}') + debug_log(f'Preview: job={shared.state.job} active={active} progress={current}/{total} step={shared.state.current_image_sampling_step}/{step_x}/{step_y} request={id_live_preview} last={shared.state.id_live_preview} enabled={shared.opts.live_previews_enable} job={shared.state.preview_job} updated={updated} image={shared.state.current_image} elapsed={elapsed:.3f}') if not active: return InternalProgressResponse(job=shared.state.job, active=active, queued=queued, paused=paused, completed=completed, id_live_preview=-1, debug=debug, textinfo="Queued..." if queued else "Waiting...") if shared.opts.live_previews_enable and (shared.state.id_live_preview != id_live_preview) and (shared.state.current_image is not None): From 346be51400ae11b938557643c4948dcaf27450a4 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 5 Jan 2025 13:22:19 -0500 Subject: [PATCH 220/249] refactor of progress monitoring Signed-off-by: Vladimir Mandic --- .eslintrc.json | 1 + CHANGELOG.md | 2 + javascript/progressBar.js | 37 ++++++++++------- modules/call_queue.py | 4 +- modules/gr_tempdir.py | 1 + modules/history.py | 1 + modules/images.py | 3 ++ modules/lora/lora_extract.py | 2 +- modules/processing.py | 7 +++- modules/processing_args.py | 3 +- modules/processing_callbacks.py | 5 ++- modules/processing_class.py | 2 +- modules/processing_diffusers.py | 22 +++++----- modules/processing_helpers.py | 23 +++++++++++ modules/progress.py | 15 ++----- modules/shared_state.py | 73 +++++++++++++++++++++++++++------ modules/ui_img2img.py | 2 +- modules/ui_models.py | 2 +- modules/ui_postprocessing.py | 2 +- modules/ui_txt2img.py | 2 +- scripts/xyz_grid.py | 7 +++- scripts/xyz_grid_draw.py | 8 ++-- scripts/xyz_grid_on.py | 33 ++++++++------- webui.py | 4 +- 24 files changed, 177 insertions(+), 84 deletions(-) diff --git a/.eslintrc.json b/.eslintrc.json index c86dbb749..4ca2afbdb 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -22,6 +22,7 @@ "default-case":"off", "no-await-in-loop":"off", "no-bitwise":"off", + "no-continue":"off", "no-confusing-arrow":"off", "no-console":"off", "no-empty":"off", diff --git a/CHANGELOG.md b/CHANGELOG.md index 0be26a812..3941eeec7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,8 @@ - add explicit detailer steps setting - **SysInfo**: - update to collected data and benchmarks +- **Progress**: + - refactored progress monitoring, job updates and live preview - **Metadata**: - improved metadata save and restore - **Fixes**: diff --git a/javascript/progressBar.js b/javascript/progressBar.js index 27404b440..0bac99d6f 100644 --- a/javascript/progressBar.js +++ b/javascript/progressBar.js @@ -20,29 +20,36 @@ function checkPaused(state) { function setProgress(res) { const elements = ['txt2img_generate', 'img2img_generate', 'extras_generate', 'control_generate']; - const progress = (res?.progress || 0); - let job = res?.job || ''; - job = job.replace('txt2img', 'Generate').replace('img2img', 'Generate'); - const perc = res && (progress > 0) ? `${Math.round(100.0 * progress)}%` : ''; - let sec = res?.eta || 0; + const progress = res?.progress || 0; + const job = res?.job || ''; + let perc = ''; let eta = ''; - if (res?.paused) eta = 'Paused'; - else if (res?.completed || (progress > 0.99)) eta = 'Finishing'; - else if (sec === 0) eta = 'Starting'; + if (job === 'VAE') perc = 'Decode'; else { - const min = Math.floor(sec / 60); - sec %= 60; - eta = min > 0 ? `${Math.round(min)}m ${Math.round(sec)}s` : `${Math.round(sec)}s`; + perc = res && (progress > 0) && (progress < 1) ? `${Math.round(100.0 * progress)}% ` : ''; + let sec = res?.eta || 0; + if (res?.paused) eta = 'Paused'; + else if (res?.completed || (progress > 0.99)) eta = 'Finishing'; + else if (sec === 0) eta = 'Start'; + else { + const min = Math.floor(sec / 60); + sec %= 60; + eta = min > 0 ? `${Math.round(min)}m ${Math.round(sec)}s` : `${Math.round(sec)}s`; + } } document.title = `SD.Next ${perc}`; for (const elId of elements) { const el = document.getElementById(elId); if (el) { - el.innerText = (res ? `${job} ${perc} ${eta}` : 'Generate'); + const jobLabel = (res ? `${job} ${perc}${eta}` : 'Generate').trim(); + el.innerText = jobLabel; if (!window.waitForUiReady) { - el.style.background = res && (progress > 0) - ? `linear-gradient(to right, var(--primary-500) 0%, var(--primary-800) ${perc}, var(--neutral-700) ${perc})` - : 'var(--button-primary-background-fill)'; + const gradient = perc !== '' ? perc : '100%'; + if (jobLabel === 'Generate') el.style.background = 'var(--primary-500)'; + else if (jobLabel.endsWith('Decode')) continue; + else if (jobLabel.endsWith('Start') || jobLabel.endsWith('Finishing')) el.style.background = 'var(--primary-800)'; + else if (res && progress > 0 && progress < 1) el.style.background = `linear-gradient(to right, var(--primary-500) 0%, var(--primary-800) ${gradient}, var(--neutral-700) ${gradient})`; + else el.style.background = 'var(--primary-500)'; } } } diff --git a/modules/call_queue.py b/modules/call_queue.py index 11ba7b56e..af6f2e4d0 100644 --- a/modules/call_queue.py +++ b/modules/call_queue.py @@ -15,8 +15,8 @@ def f(*args, **kwargs): return f -def wrap_gradio_gpu_call(func, extra_outputs=None): - name = func.__name__ +def wrap_gradio_gpu_call(func, extra_outputs=None, name=None): + name = name or func.__name__ def f(*args, **kwargs): # if the first argument is a string that says "task(...)", it is treated as a job id if len(args) > 0 and type(args[0]) == str and args[0][0:5] == "task(" and args[0][-1] == ")": diff --git a/modules/gr_tempdir.py b/modules/gr_tempdir.py index 9076ad407..bbe2b2192 100644 --- a/modules/gr_tempdir.py +++ b/modules/gr_tempdir.py @@ -71,6 +71,7 @@ def pil_to_temp_file(self, img: Image, dir: str, format="png") -> str: # pylint: img.already_saved_as = name size = os.path.getsize(name) shared.log.debug(f'Save temp: image="{name}" width={img.width} height={img.height} size={size}') + shared.state.image_history += 1 params = ', '.join([f'{k}: {v}' for k, v in img.info.items()]) params = params[12:] if params.startswith('parameters: ') else params with open(os.path.join(paths.data_path, "params.txt"), "w", encoding="utf8") as file: diff --git a/modules/history.py b/modules/history.py index 63c1669c2..f60395edd 100644 --- a/modules/history.py +++ b/modules/history.py @@ -62,6 +62,7 @@ def find(self, name): return -1 def add(self, latent, preview=None, info=None, ops=[]): + shared.state.latent_history += 1 if shared.opts.latent_history == 0: return if torch.is_tensor(latent): diff --git a/modules/images.py b/modules/images.py index 657a51193..c3a8cee54 100644 --- a/modules/images.py +++ b/modules/images.py @@ -29,6 +29,7 @@ def atomically_save_image(): Image.MAX_IMAGE_PIXELS = None # disable check in Pillow and rely on check below to allow large custom image sizes while True: image, filename, extension, params, exifinfo, filename_txt = save_queue.get() + shared.state.image_history += 1 with open(os.path.join(paths.data_path, "params.txt"), "w", encoding="utf8") as file: file.write(exifinfo) fn = filename + extension @@ -49,6 +50,7 @@ def atomically_save_image(): shared.log.info(f'Save: text="{filename_txt}" len={len(exifinfo)}') except Exception as e: shared.log.warning(f'Save failed: description={filename_txt} {e}') + # actual save if image_format == 'PNG': pnginfo_data = PngImagePlugin.PngInfo() @@ -87,6 +89,7 @@ def atomically_save_image(): errors.display(e, 'Image save') size = os.path.getsize(fn) if os.path.exists(fn) else 0 shared.log.info(f'Save: image="{fn}" type={image_format} width={image.width} height={image.height} size={size}') + if shared.opts.save_log_fn != '' and len(exifinfo) > 0: fn = os.path.join(paths.data_path, shared.opts.save_log_fn) if not fn.endswith('.json'): diff --git a/modules/lora/lora_extract.py b/modules/lora/lora_extract.py index 58cd065bb..a226b6017 100644 --- a/modules/lora/lora_extract.py +++ b/modules/lora/lora_extract.py @@ -265,7 +265,7 @@ def gr_show(visible=True): auto_rank.change(fn=lambda x: gr_show(x), inputs=[auto_rank], outputs=[rank_ratio]) extract.click( - fn=wrap_gradio_gpu_call(make_lora, extra_outputs=[]), + fn=wrap_gradio_gpu_call(make_lora, extra_outputs=[], name='LoRA'), inputs=[filename, rank, auto_rank, rank_ratio, modules, overwrite], outputs=[status] ) diff --git a/modules/processing.py b/modules/processing.py index 23fde5dee..c2221823e 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -280,19 +280,22 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: output_images = [] process_init(p) - if os.path.exists(shared.opts.embeddings_dir) and not p.do_not_reload_embeddings and not shared.native: + if not shared.native and os.path.exists(shared.opts.embeddings_dir) and not p.do_not_reload_embeddings: modules.sd_hijack.model_hijack.embedding_db.load_textual_inversion_embeddings(force_reload=False) if p.scripts is not None and isinstance(p.scripts, scripts.ScriptRunner): p.scripts.process(p) ema_scope_context = p.sd_model.ema_scope if not shared.native else nullcontext - shared.state.job_count = p.n_iter + if not shared.native: + shared.state.job_count = p.n_iter with devices.inference_context(), ema_scope_context(): t0 = time.time() if not hasattr(p, 'skip_init'): p.init(p.all_prompts, p.all_seeds, p.all_subseeds) debug(f'Processing inner: args={vars(p)}') for n in range(p.n_iter): + # if hasattr(p, 'skip_processing'): + # continue pag.apply(p) debug(f'Processing inner: iteration={n+1}/{p.n_iter}') p.iteration = n diff --git a/modules/processing_args.py b/modules/processing_args.py index 529971d5c..7b27e8c31 100644 --- a/modules/processing_args.py +++ b/modules/processing_args.py @@ -15,6 +15,7 @@ debug_enabled = os.environ.get('SD_DIFFUSERS_DEBUG', None) debug_log = shared.log.trace if os.environ.get('SD_DIFFUSERS_DEBUG', None) is not None else lambda *args, **kwargs: None +disable_pbar = os.environ.get('SD_DISABLE_PBAR', None) is not None def task_specific_kwargs(p, model): @@ -107,7 +108,7 @@ def set_pipeline_args(p, model, prompts:list, negative_prompts:list, prompts_2:t shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) apply_circular(p.tiling, model) if hasattr(model, "set_progress_bar_config"): - model.set_progress_bar_config(bar_format='Progress {rate_fmt}{postfix} {bar} {percentage:3.0f}% {n_fmt}/{total_fmt} {elapsed} {remaining} ' + '\x1b[38;5;71m' + desc, ncols=80, colour='#327fba') + model.set_progress_bar_config(bar_format='Progress {rate_fmt}{postfix} {bar} {percentage:3.0f}% {n_fmt}/{total_fmt} {elapsed} {remaining} ' + '\x1b[38;5;71m' + desc, ncols=80, colour='#327fba', disable=disable_pbar) args = {} has_vae = hasattr(model, 'vae') or (hasattr(model, 'pipe') and hasattr(model.pipe, 'vae')) if hasattr(model, 'pipe') and not hasattr(model, 'no_recurse'): # recurse diff --git a/modules/processing_callbacks.py b/modules/processing_callbacks.py index d3e57818b..0191eff72 100644 --- a/modules/processing_callbacks.py +++ b/modules/processing_callbacks.py @@ -56,8 +56,9 @@ def diffusers_callback(pipe, step: int = 0, timestep: int = 0, kwargs: dict = {} latents = kwargs.get('latents', None) if debug: debug_callback(f'Callback: step={step} timestep={timestep} latents={latents.shape if latents is not None else None} kwargs={list(kwargs)}') - order = getattr(pipe.scheduler, "order", 1) if hasattr(pipe, 'scheduler') else 1 - shared.state.sampling_step = step // order + shared.state.step() + # order = getattr(pipe.scheduler, "order", 1) if hasattr(pipe, 'scheduler') else 1 + # shared.state.sampling_step = step // order if shared.state.interrupted or shared.state.skipped: raise AssertionError('Interrupted...') if shared.state.paused: diff --git a/modules/processing_class.py b/modules/processing_class.py index 5960fea76..60aa52c50 100644 --- a/modules/processing_class.py +++ b/modules/processing_class.py @@ -581,7 +581,7 @@ def init_hr(self, scale = None, upscaler = None, force = False): else: self.hr_upscale_to_x, self.hr_upscale_to_y = self.hr_resize_x, self.hr_resize_y # hypertile_set(self, hr=True) - shared.state.job_count = 2 * self.n_iter + # shared.state.job_count = 2 * self.n_iter shared.log.debug(f'Control hires: upscaler="{self.hr_upscaler}" scale={scale} fixed={not use_scale} size={self.hr_upscale_to_x}x{self.hr_upscale_to_y}') diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index d978cbe2b..01a3dae0b 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -6,7 +6,7 @@ import torchvision.transforms.functional as TF from PIL import Image from modules import shared, devices, processing, sd_models, errors, sd_hijack_hypertile, processing_vae, sd_models_compile, hidiffusion, timer, modelstats, extra_networks -from modules.processing_helpers import resize_hires, calculate_base_steps, calculate_hires_steps, calculate_refiner_steps, save_intermediate, update_sampler, is_txt2img, is_refiner_enabled +from modules.processing_helpers import resize_hires, calculate_base_steps, calculate_hires_steps, calculate_refiner_steps, save_intermediate, update_sampler, is_txt2img, is_refiner_enabled, get_job_name from modules.processing_args import set_pipeline_args from modules.onnx_impl import preprocess_pipeline as preprocess_onnx_pipeline, check_parameters_changed as olive_check_parameters_changed from modules.lora import networks @@ -53,8 +53,9 @@ def restore_state(p: processing.StableDiffusionProcessing): def process_base(p: processing.StableDiffusionProcessing): - use_refiner_start = is_txt2img() and is_refiner_enabled(p) and not p.is_hr_pass and p.refiner_start > 0 and p.refiner_start < 1 - use_denoise_start = not is_txt2img() and p.refiner_start > 0 and p.refiner_start < 1 + txt2img = is_txt2img() + use_refiner_start = txt2img and is_refiner_enabled(p) and not p.is_hr_pass and p.refiner_start > 0 and p.refiner_start < 1 + use_denoise_start = not txt2img and p.refiner_start > 0 and p.refiner_start < 1 shared.sd_model = update_pipeline(shared.sd_model, p) update_sampler(p, shared.sd_model) @@ -76,7 +77,8 @@ def process_base(p: processing.StableDiffusionProcessing): clip_skip=p.clip_skip, desc='Base', ) - shared.state.sampling_steps = base_args.get('prior_num_inference_steps', None) or p.steps or base_args.get('num_inference_steps', None) + base_steps = base_args.get('prior_num_inference_steps', None) or p.steps or base_args.get('num_inference_steps', None) + shared.state.update(get_job_name(p, shared.sd_model), base_steps, 1) if shared.opts.scheduler_eta is not None and shared.opts.scheduler_eta > 0 and shared.opts.scheduler_eta < 1: p.extra_generation_params["Sampler Eta"] = shared.opts.scheduler_eta output = None @@ -172,7 +174,7 @@ def process_hires(p: processing.StableDiffusionProcessing, output): p.ops.append('upscale') if shared.opts.samples_save and not p.do_not_save_samples and shared.opts.save_images_before_highres_fix and hasattr(shared.sd_model, 'vae'): save_intermediate(p, latents=output.images, suffix="-before-hires") - shared.state.job = 'Upscale' + shared.state.update('Upscale', 0, 1) output.images = resize_hires(p, latents=output.images) sd_hijack_hypertile.hypertile_set(p, hr=True) @@ -190,7 +192,6 @@ def process_hires(p: processing.StableDiffusionProcessing, output): shared.log.warning('HiRes skip: denoising=0') p.hr_force = False if p.hr_force: - shared.state.job_count = 2 * p.n_iter shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE) if 'Upscale' in shared.sd_model.__class__.__name__ or 'Flux' in shared.sd_model.__class__.__name__ or 'Kandinsky' in shared.sd_model.__class__.__name__: output.images = processing_vae.vae_decode(latents=output.images, model=shared.sd_model, full_quality=p.full_quality, output_type='pil', width=p.width, height=p.height) @@ -217,8 +218,8 @@ def process_hires(p: processing.StableDiffusionProcessing, output): strength=strength, desc='Hires', ) - shared.state.job = 'HiRes' - shared.state.sampling_steps = hires_args.get('prior_num_inference_steps', None) or p.steps or hires_args.get('num_inference_steps', None) + hires_steps = hires_args.get('prior_num_inference_steps', None) or p.hr_second_pass_steps or hires_args.get('num_inference_steps', None) + shared.state.update(get_job_name(p, shared.sd_model), hires_steps, 1) try: shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) sd_models.move_model(shared.sd_model, devices.device) @@ -255,8 +256,6 @@ def process_refine(p: processing.StableDiffusionProcessing, output): # optional refiner pass or decode if is_refiner_enabled(p): prev_job = shared.state.job - shared.state.job = 'Refine' - shared.state.job_count +=1 if shared.opts.samples_save and not p.do_not_save_samples and shared.opts.save_images_before_refiner and hasattr(shared.sd_model, 'vae'): save_intermediate(p, latents=output.images, suffix="-before-refiner") if shared.opts.diffusers_move_base: @@ -306,7 +305,8 @@ def process_refine(p: processing.StableDiffusionProcessing, output): prompt_attention='fixed', desc='Refiner', ) - shared.state.sampling_steps = refiner_args.get('prior_num_inference_steps', None) or p.steps or refiner_args.get('num_inference_steps', None) + refiner_steps = refiner_args.get('prior_num_inference_steps', None) or p.steps or refiner_args.get('num_inference_steps', None) + shared.state.update(get_job_name(p, shared.sd_refiner), refiner_steps, 1) try: if 'requires_aesthetics_score' in shared.sd_refiner.config: # sdxl-model needs false and sdxl-refiner needs true shared.sd_refiner.register_to_config(requires_aesthetics_score = getattr(shared.sd_refiner, 'tokenizer', None) is None) diff --git a/modules/processing_helpers.py b/modules/processing_helpers.py index 51cbcff7f..fe6fa3b3c 100644 --- a/modules/processing_helpers.py +++ b/modules/processing_helpers.py @@ -584,3 +584,26 @@ def update_sampler(p, sd_model, second_pass=False): sampler_options.append('low order') if len(sampler_options) > 0: p.extra_generation_params['Sampler options'] = '/'.join(sampler_options) + + +def get_job_name(p, model): + if hasattr(model, 'pipe'): + model = model.pipe + if hasattr(p, 'xyz'): + return 'Ignore' # xyz grid handles its own jobs + if sd_models.get_diffusers_task(model) == sd_models.DiffusersTaskType.TEXT_2_IMAGE: + return 'Text' + elif sd_models.get_diffusers_task(model) == sd_models.DiffusersTaskType.IMAGE_2_IMAGE: + if p.is_refiner_pass: + return 'Refiner' + elif p.is_hr_pass: + return 'Hires' + else: + return 'Image' + elif sd_models.get_diffusers_task(model) == sd_models.DiffusersTaskType.INPAINTING: + if p.detailer: + return 'Detailer' + else: + return 'Inpaint' + else: + return 'Unknown' diff --git a/modules/progress.py b/modules/progress.py index f6b47f1c0..8b15f7e13 100644 --- a/modules/progress.py +++ b/modules/progress.py @@ -64,23 +64,16 @@ def progressapi(req: ProgressRequest): queued = req.id_task in pending_tasks completed = req.id_task in finished_tasks paused = shared.state.paused - shared.state.job_count = max(shared.state.frame_count, shared.state.job_count, shared.state.job_no) - batch_x = max(shared.state.job_no, 0) - batch_y = max(shared.state.job_count, 1) - step_x = max(shared.state.sampling_step, 0) - step_y = max(shared.state.sampling_steps, 1) - current = step_y * batch_x + step_x - total = step_y * batch_y - while total < current: - total += step_y - progress = min(1, abs(current / total) if total > 0 else 0) + step = max(shared.state.sampling_step, 0) + steps = max(shared.state.sampling_steps, 1) + progress = round(min(1, abs(step / steps) if steps > 0 else 0), 2) elapsed = time.time() - shared.state.time_start if shared.state.time_start is not None else 0 predicted = elapsed / progress if progress > 0 else None eta = predicted - elapsed if predicted is not None else None id_live_preview = req.id_live_preview live_preview = None updated = shared.state.set_current_image() - debug_log(f'Preview: job={shared.state.job} active={active} progress={current}/{total} step={shared.state.current_image_sampling_step}/{step_x}/{step_y} request={id_live_preview} last={shared.state.id_live_preview} enabled={shared.opts.live_previews_enable} job={shared.state.preview_job} updated={updated} image={shared.state.current_image} elapsed={elapsed:.3f}') + debug_log(f'Preview: job={shared.state.job} active={active} progress={step}/{steps}/{progress} image={shared.state.current_image_sampling_step} request={id_live_preview} last={shared.state.id_live_preview} enabled={shared.opts.live_previews_enable} job={shared.state.preview_job} updated={updated} image={shared.state.current_image} elapsed={elapsed:.3f}') if not active: return InternalProgressResponse(job=shared.state.job, active=active, queued=queued, paused=paused, completed=completed, id_live_preview=-1, debug=debug, textinfo="Queued..." if queued else "Waiting...") if shared.opts.live_previews_enable and (shared.state.id_live_preview != id_live_preview) and (shared.state.current_image is not None): diff --git a/modules/shared_state.py b/modules/shared_state.py index 9f56b14e0..b4c92bb65 100644 --- a/modules/shared_state.py +++ b/modules/shared_state.py @@ -1,10 +1,18 @@ import os +import sys import time import datetime from modules.errors import log, display +debug_output = os.environ.get('SD_STATE_DEBUG', None) + + class State: + job_history = [] + task_history = [] + image_history = 0 + latent_history = 0 skipped = False interrupted = False paused = False @@ -14,7 +22,7 @@ class State: frame_count = 0 total_jobs = 0 job_timestamp = '0' - sampling_step = 0 + _sampling_step = 0 sampling_steps = 0 current_latent = None current_noise_pred = None @@ -32,29 +40,48 @@ class State: need_restart = False server_start = time.time() oom = False - debug_output = os.environ.get('SD_STATE_DEBUG', None) def __str__(self) -> str: - return f'State: job={self.job} {self.job_no}/{self.job_count} step={self.sampling_step}/{self.sampling_steps} skipped={self.skipped} interrupted={self.interrupted} paused={self.paused} info={self.textinfo}' + status = ' ' + status += 'skipped ' if self.skipped else '' + status += 'interrupted ' if self.interrupted else '' + status += 'paused ' if self.paused else '' + status += 'restart ' if self.need_restart else '' + status += 'oom ' if self.oom else '' + status += 'api ' if self.api else '' + fn = f'{sys._getframe(3).f_code.co_name}:{sys._getframe(2).f_code.co_name}' # pylint: disable=protected-access + return f'State: ts={self.job_timestamp} job={self.job} jobs={self.job_no+1}/{self.job_count}/{self.total_jobs} step={self.sampling_step}/{self.sampling_steps} preview={self.preview_job}/{self.id_live_preview}/{self.current_image_sampling_step} status="{status.strip()}" fn={fn}' + + @property + def sampling_step(self): + return self._sampling_step + + @sampling_step.setter + def sampling_step(self, value): + self._sampling_step = value + if debug_output: + log.trace(f'State step: {self}') def skip(self): - log.debug('Requested skip') + log.debug('State: skip requested') self.skipped = True def interrupt(self): - log.debug('Requested interrupt') + log.debug('State: interrupt requested') self.interrupted = True def pause(self): self.paused = not self.paused - log.debug(f'Requested {"pause" if self.paused else "continue"}') + log.debug(f'State: {"pause" if self.paused else "continue"} requested') def nextjob(self): import modules.devices self.do_set_current_image() self.job_no += 1 - self.sampling_step = 0 + # self.sampling_step = 0 self.current_image_sampling_step = 0 + if debug_output: + log.trace(f'State next: {self}') modules.devices.torch_gc() def dict(self): @@ -104,6 +131,7 @@ def status(self): def begin(self, title="", api=None): import modules.devices + self.job_history.append(title) self.total_jobs += 1 self.current_image = None self.current_image_sampling_step = 0 @@ -115,19 +143,20 @@ def begin(self, title="", api=None): self.interrupted = False self.preview_job = -1 self.job = title - self.job_count = -1 - self.frame_count = -1 + self.job_count = 0 + self.frame_count = 0 self.job_no = 0 self.job_timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S") self.paused = False - self.sampling_step = 0 + self._sampling_step = 0 + self.sampling_steps = 0 self.skipped = False self.textinfo = None self.prediction_type = "epsilon" self.api = api or self.api self.time_start = time.time() - if self.debug_output: - log.debug(f'State begin: {self.job}') + if debug_output: + log.trace(f'State begin: {self}') modules.devices.torch_gc() def end(self, api=None): @@ -136,6 +165,8 @@ def end(self, api=None): # fn = f'{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access # log.debug(f'Access state.end: {fn}') # pylint: disable=protected-access self.time_start = time.time() + if debug_output: + log.trace(f'State end: {self}') self.job = "" self.job_count = 0 self.job_no = 0 @@ -147,6 +178,24 @@ def end(self, api=None): self.api = api or self.api modules.devices.torch_gc() + def step(self, step:int=1): + self.sampling_step += step + + def update(self, job:str, steps:int=0, jobs:int=0): + self.task_history.append(job) + # self._sampling_step = 0 + if job == 'Ignore': + return + elif job == 'Grid': + self.sampling_steps = steps + self.job_count = jobs + else: + self.sampling_steps += steps * jobs + self.job_count += jobs + self.job = job + if debug_output: + log.trace(f'State update: {self} steps={steps} jobs={jobs}') + def set_current_image(self): if self.job == 'VAE' or self.job == 'Upscale': # avoid generating preview while vae is running return False diff --git a/modules/ui_img2img.py b/modules/ui_img2img.py index 48f22af7e..74bfd0519 100644 --- a/modules/ui_img2img.py +++ b/modules/ui_img2img.py @@ -193,7 +193,7 @@ def select_img2img_tab(tab): override_settings, ] img2img_dict = dict( - fn=wrap_gradio_gpu_call(modules.img2img.img2img, extra_outputs=[None, '', '']), + fn=wrap_gradio_gpu_call(modules.img2img.img2img, extra_outputs=[None, '', ''], name='Image'), _js="submit_img2img", inputs= img2img_args + img2img_script_inputs, outputs=[ diff --git a/modules/ui_models.py b/modules/ui_models.py index 7ab8b0d07..5d5b452e2 100644 --- a/modules/ui_models.py +++ b/modules/ui_models.py @@ -290,7 +290,7 @@ def preset_choices(sdxl): beta_apply_preset.click(fn=load_presets, inputs=[beta_preset, beta_preset_lambda], outputs=[beta_base, beta_in_blocks, beta_mid_block, beta_out_blocks, tabs]) modelmerger_merge.click( - fn=wrap_gradio_gpu_call(modelmerger, extra_outputs=lambda: [gr.update() for _ in range(4)]), + fn=wrap_gradio_gpu_call(modelmerger, extra_outputs=lambda: [gr.update() for _ in range(4)], name='Models'), _js='modelmerger', inputs=[ dummy_component, diff --git a/modules/ui_postprocessing.py b/modules/ui_postprocessing.py index e9ac2d72a..6e12339e7 100644 --- a/modules/ui_postprocessing.py +++ b/modules/ui_postprocessing.py @@ -129,7 +129,7 @@ def create_ui(): ) submit.click( _js="submit_postprocessing", - fn=call_queue.wrap_gradio_gpu_call(submit_process, extra_outputs=[None, '']), + fn=call_queue.wrap_gradio_gpu_call(submit_process, extra_outputs=[None, ''], name='Postprocess'), inputs=[ tab_index, extras_image, diff --git a/modules/ui_txt2img.py b/modules/ui_txt2img.py index be27b7e9e..9fc69a4db 100644 --- a/modules/ui_txt2img.py +++ b/modules/ui_txt2img.py @@ -77,7 +77,7 @@ def create_ui(): override_settings, ] txt2img_dict = dict( - fn=wrap_gradio_gpu_call(modules.txt2img.txt2img, extra_outputs=[None, '', '']), + fn=wrap_gradio_gpu_call(modules.txt2img.txt2img, extra_outputs=[None, '', ''], name='Text'), _js="submit_txt2img", inputs=txt2img_args + txt2img_script_inputs, outputs=[ diff --git a/scripts/xyz_grid.py b/scripts/xyz_grid.py index bb067ea21..8bf149777 100644 --- a/scripts/xyz_grid.py +++ b/scripts/xyz_grid.py @@ -253,6 +253,7 @@ def fix_axis_seeds(axis_opt, axis_list): ys = fix_axis_seeds(y_opt, ys) zs = fix_axis_seeds(z_opt, zs) + total_jobs = len(xs) * len(ys) * len(zs) if x_opt.label == 'Steps': total_steps = sum(xs) * len(ys) * len(zs) elif y_opt.label == 'Steps': @@ -260,7 +261,7 @@ def fix_axis_seeds(axis_opt, axis_list): elif z_opt.label == 'Steps': total_steps = sum(zs) * len(xs) * len(ys) else: - total_steps = p.steps * len(xs) * len(ys) * len(zs) + total_steps = p.steps * total_jobs if isinstance(p, processing.StableDiffusionProcessingTxt2Img) and p.enable_hr: if x_opt.label == "Hires steps": total_steps += sum(xs) * len(ys) * len(zs) @@ -269,10 +270,12 @@ def fix_axis_seeds(axis_opt, axis_list): elif z_opt.label == "Hires steps": total_steps += sum(zs) * len(xs) * len(ys) elif p.hr_second_pass_steps: - total_steps += p.hr_second_pass_steps * len(xs) * len(ys) * len(zs) + total_steps += p.hr_second_pass_steps * total_jobs else: total_steps *= 2 total_steps *= p.n_iter + shared.state.update('Grid', total_steps, total_jobs * p.n_iter) + image_cell_count = p.n_iter * p.batch_size shared.log.info(f"XYZ grid: images={len(xs)*len(ys)*len(zs)*image_cell_count} grid={len(zs)} shape={len(xs)}x{len(ys)} cells={len(zs)} steps={total_steps}") AxisInfo = namedtuple('AxisInfo', ['axis', 'values']) diff --git a/scripts/xyz_grid_draw.py b/scripts/xyz_grid_draw.py index cd7eb8d1f..bac96bb8b 100644 --- a/scripts/xyz_grid_draw.py +++ b/scripts/xyz_grid_draw.py @@ -10,7 +10,7 @@ def draw_xyz_grid(p, xs, ys, zs, x_labels, y_labels, z_labels, cell, draw_legend z_texts = [[images.GridAnnotation(z)] for z in z_labels] list_size = (len(xs) * len(ys) * len(zs)) processed_result = None - shared.state.job_count = list_size * p.n_iter + t0 = time.time() i = 0 @@ -22,7 +22,6 @@ def process_cell(x, y, z, ix, iy, iz): def index(ix, iy, iz): return ix + iy * len(xs) + iz * len(xs) * len(ys) - shared.state.job = 'Grid' p0 = time.time() processed: processing.Processed = cell(x, y, z, ix, iy, iz) p1 = time.time() @@ -63,7 +62,7 @@ def index(ix, iy, iz): cell_mode = processed_result.images[0].mode cell_size = processed_result.images[0].size processed_result.images[idx] = Image.new(cell_mode, cell_size) - return + shared.state.nextjob() if first_axes_processed == 'x': for ix, x in enumerate(xs): @@ -129,5 +128,6 @@ def index(ix, iy, iz): processed_result.infotexts.insert(0, processed_result.infotexts[0]) t2 = time.time() - shared.log.info(f'XYZ grid complete: images={list_size} size={grid.size if grid is not None else None} time={t1-t0:.2f} save={t2-t1:.2f}') + shared.log.info(f'XYZ grid complete: images={list_size} results={len(processed_result.images)}size={grid.size if grid is not None else None} time={t1-t0:.2f} save={t2-t1:.2f}') + p.skip_processing = True return processed_result diff --git a/scripts/xyz_grid_on.py b/scripts/xyz_grid_on.py index 0abd0fa1c..5c5ef8bf5 100644 --- a/scripts/xyz_grid_on.py +++ b/scripts/xyz_grid_on.py @@ -18,7 +18,7 @@ active = False -cache = None +xyz_results_cache = None debug = shared.log.trace if os.environ.get('SD_XYZ_DEBUG', None) is not None else lambda *args, **kwargs: None @@ -188,8 +188,8 @@ def process(self, p, include_time, include_text, margin_size, create_video, video_type, video_duration, video_loop, video_pad, video_interpolate, ): # pylint: disable=W0221 - global active, cache # pylint: disable=W0603 - cache = None + global active, xyz_results_cache # pylint: disable=W0603 + xyz_results_cache = None if not enabled or active: return active = True @@ -266,6 +266,7 @@ def fix_axis_seeds(axis_opt, axis_list): ys = fix_axis_seeds(y_opt, ys) zs = fix_axis_seeds(z_opt, zs) + total_jobs = len(xs) * len(ys) * len(zs) if x_opt.label == 'Steps': total_steps = sum(xs) * len(ys) * len(zs) elif y_opt.label == 'Steps': @@ -273,8 +274,8 @@ def fix_axis_seeds(axis_opt, axis_list): elif z_opt.label == 'Steps': total_steps = sum(zs) * len(xs) * len(ys) else: - total_steps = p.steps * len(xs) * len(ys) * len(zs) - if isinstance(p, processing.StableDiffusionProcessingTxt2Img) and p.enable_hr: + total_steps = p.steps * total_jobs + if p.enable_hr: if x_opt.label == "Hires steps": total_steps += sum(xs) * len(ys) * len(zs) elif y_opt.label == "Hires steps": @@ -282,10 +283,16 @@ def fix_axis_seeds(axis_opt, axis_list): elif z_opt.label == "Hires steps": total_steps += sum(zs) * len(xs) * len(ys) elif p.hr_second_pass_steps: - total_steps += p.hr_second_pass_steps * len(xs) * len(ys) * len(zs) + total_steps += p.hr_second_pass_steps * total_jobs else: total_steps *= 2 + if p.detailer: + total_steps += shared.opts.detailer_steps * total_jobs + total_steps *= p.n_iter + total_jobs *= p.n_iter + shared.state.update('Grid', total_steps, total_jobs) + image_cell_count = p.n_iter * p.batch_size shared.log.info(f"XYZ grid start: images={len(xs)*len(ys)*len(zs)*image_cell_count} grid={len(zs)} shape={len(xs)}x{len(ys)} cells={len(zs)} steps={total_steps}") AxisInfo = namedtuple('AxisInfo', ['axis', 'values']) @@ -360,7 +367,7 @@ def cell(x, y, z, ix, iy, iz): return processed with SharedSettingsStackHelper(): - processed = draw_xyz_grid( + processed: processing.Processed = draw_xyz_grid( p, xs=xs, ys=ys, @@ -418,19 +425,15 @@ def cell(x, y, z, ix, iy, iz): p.do_not_save_samples = True p.disable_extra_networks = True active = False - cache = processed + xyz_results_cache = processed return processed def process_images(self, p, *args): # pylint: disable=W0221, W0613 - if hasattr(cache, 'used'): - cache.images.clear() - cache.used = False - elif cache is not None and len(cache.images) > 0: - cache.used = True + if xyz_results_cache is not None and len(xyz_results_cache.images) > 0: p.restore_faces = False p.detailer = False p.color_corrections = None - p.scripts = None - return cache + # p.scripts = None + return xyz_results_cache return None diff --git a/webui.py b/webui.py index 3c8361ecf..33769994e 100644 --- a/webui.py +++ b/webui.py @@ -1,6 +1,7 @@ import io import os import sys +import time import glob import signal import asyncio @@ -156,6 +157,7 @@ def initialize(): # make the program just exit at ctrl+c without waiting for anything def sigint_handler(_sig, _frame): + log.trace(f'State history: uptime={round(time.time() - shared.state.server_start)} jobs={len(shared.state.job_history)} tasks={len(shared.state.task_history)} latents={shared.state.latent_history} images={shared.state.image_history}') log.info('Exiting') try: for f in glob.glob("*.lock"): @@ -176,9 +178,9 @@ def load_model(): thread_model.start() thread_refiner = Thread(target=lambda: shared.sd_refiner) thread_refiner.start() - shared.state.end() thread_model.join() thread_refiner.join() + shared.state.end() timer.startup.record("checkpoint") shared.opts.onchange("sd_model_checkpoint", wrap_queued_call(lambda: modules.sd_models.reload_model_weights(op='model')), call=False) shared.opts.onchange("sd_model_refiner", wrap_queued_call(lambda: modules.sd_models.reload_model_weights(op='refiner')), call=False) From 336564251a419cdf43df3bdd5f43049e6d6c4fb7 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 5 Jan 2025 14:02:22 -0500 Subject: [PATCH 221/249] fix controlnet with hires Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 1 + modules/processing_args.py | 1 + modules/processing_diffusers.py | 1 + 3 files changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3941eeec7..f3b52726a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ - sd35 img2img - samplers test for scale noise before using - scheduler api + - controlnet with hires ## Update for 2024-12-31 diff --git a/modules/processing_args.py b/modules/processing_args.py index 7b27e8c31..774c5db64 100644 --- a/modules/processing_args.py +++ b/modules/processing_args.py @@ -294,6 +294,7 @@ def set_pipeline_args(p, model, prompts:list, negative_prompts:list, prompts_2:t debug_log(f'Diffusers task args: {task_args}') for k, v in task_args.items(): if k in possible: + print('HERE OVERRIDE', k, v) args[k] = v else: debug_log(f'Diffusers unknown task args: {k}={v}') diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 01a3dae0b..b5a0194c2 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -201,6 +201,7 @@ def process_hires(p: processing.StableDiffusionProcessing, output): update_sampler(p, shared.sd_model, second_pass=True) orig_denoise = p.denoising_strength p.denoising_strength = strength + p.task_args.pop('image', None) # remove image override from hires hires_args = set_pipeline_args( p=p, model=shared.sd_model, From de679eb886f5a02fa48cc059dbd26c0a276b1ff9 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 5 Jan 2025 14:50:52 -0500 Subject: [PATCH 222/249] remove legacy restore resolution Signed-off-by: Vladimir Mandic --- modules/generation_parameters_copypaste.py | 8 +++++--- modules/infotext.py | 2 +- modules/processing_args.py | 1 - modules/shared.py | 4 ++-- modules/ui_control.py | 7 +++++-- 5 files changed, 13 insertions(+), 9 deletions(-) diff --git a/modules/generation_parameters_copypaste.py b/modules/generation_parameters_copypaste.py index ea53c6b23..ce1896169 100644 --- a/modules/generation_parameters_copypaste.py +++ b/modules/generation_parameters_copypaste.py @@ -124,12 +124,11 @@ def connect_paste_params_buttons(): if binding.tabname not in paste_fields: debug(f"Not not registered: tab={binding.tabname}") continue + """ + # legacy code that sets width/height based on image itself instead of metadata destination_image_component = paste_fields[binding.tabname]["init_img"] - fields = paste_fields[binding.tabname]["fields"] - override_settings_component = binding.override_settings_component or paste_fields[binding.tabname]["override_settings_component"] destination_width_component = next(iter([field for field, name in fields if name == "Size-1"] if fields else []), None) destination_height_component = next(iter([field for field, name in fields if name == "Size-2"] if fields else []), None) - if binding.source_image_component and destination_image_component: if isinstance(binding.source_image_component, gr.Gallery): func = send_image_and_dimensions if destination_width_component else image_from_url_text @@ -144,6 +143,9 @@ def connect_paste_params_buttons(): outputs=[destination_image_component, destination_width_component, destination_height_component] if destination_width_component else [destination_image_component], show_progress=False, ) + """ + fields = paste_fields[binding.tabname]["fields"] + override_settings_component = binding.override_settings_component or paste_fields[binding.tabname]["override_settings_component"] if binding.source_text_component is not None and fields is not None: connect_paste(binding.paste_button, fields, binding.source_text_component, override_settings_component, binding.tabname) if binding.source_tabname is not None and fields is not None and binding.source_tabname in paste_fields: diff --git a/modules/infotext.py b/modules/infotext.py index baa995c88..017691681 100644 --- a/modules/infotext.py +++ b/modules/infotext.py @@ -108,7 +108,7 @@ def parse(infotext): params[f"{key}-2"] = int(size.group(2)) elif isinstance(params[key], str): params[key] = val - debug(f'Param parsed: type={type(params[key])} {key}={params[key]} raw="{val}"') + debug(f'Param parsed: type={type(params[key])} "{key}"={params[key]} raw="{val}"') # check_lora(params) return params diff --git a/modules/processing_args.py b/modules/processing_args.py index 774c5db64..7b27e8c31 100644 --- a/modules/processing_args.py +++ b/modules/processing_args.py @@ -294,7 +294,6 @@ def set_pipeline_args(p, model, prompts:list, negative_prompts:list, prompts_2:t debug_log(f'Diffusers task args: {task_args}') for k, v in task_args.items(): if k in possible: - print('HERE OVERRIDE', k, v) args[k] = v else: debug_log(f'Diffusers unknown task args: {k}={v}') diff --git a/modules/shared.py b/modules/shared.py index afce4266a..fd05f2e44 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -784,8 +784,8 @@ def get_default_modes(): "return_mask": OptionInfo(False, "Inpainting include greyscale mask in results"), "return_mask_composite": OptionInfo(False, "Inpainting include masked composite in results"), "disable_weights_auto_swap": OptionInfo(True, "Do not change selected model when reading generation parameters"), - "send_seed": OptionInfo(True, "Send seed when sending prompt or image to other interface"), - "send_size": OptionInfo(True, "Send size when sending prompt or image to another interface"), + "send_seed": OptionInfo(True, "Send seed when sending prompt or image to other interface", gr.Checkbox, {"visible": False}), + "send_size": OptionInfo(False, "Send size when sending prompt or image to another interface", gr.Checkbox, {"visible": False}), "quicksettings_list": OptionInfo(["sd_model_checkpoint"], "Quicksettings list", gr.Dropdown, lambda: {"multiselect":True, "choices": list(opts.data_labels.keys())}), })) diff --git a/modules/ui_control.py b/modules/ui_control.py index 4dbd1b089..a18d2305c 100644 --- a/modules/ui_control.py +++ b/modules/ui_control.py @@ -658,13 +658,16 @@ def create_ui(_blocks: gr.Blocks=None): # second pass (enable_hr, "Second pass"), (enable_hr, "Refine"), - (hr_sampler_index, "Hires sampler"), - (denoising_strength, "Hires strength"), (denoising_strength, "Denoising strength"), + (denoising_strength, "Hires strength"), + (hr_sampler_index, "Hires sampler"), + (hr_resize_mode, "Hires mode"), + (hr_resize_context, "Hires context"), (hr_upscaler, "Hires upscaler"), (hr_force, "Hires force"), (hr_second_pass_steps, "Hires steps"), (hr_scale, "Hires upscale"), + (hr_scale, "Hires scale"), (hr_resize_x, "Hires fixed-1"), (hr_resize_y, "Hires fixed-2"), # refiner From fca0addf047f1d3d1f6c888daa39b5d2a949b2ae Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Sun, 5 Jan 2025 15:12:20 -0500 Subject: [PATCH 223/249] fix controlnet with batch count Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 6 +++--- modules/processing_diffusers.py | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f3b52726a..fd44bd9de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Change Log for SD.Next -## Update for 2025-01-04 +## Update for 2025-01-05 - [Allegro Video](https://huggingface.co/rhymes-ai/Allegro) - optimizations: full offload and quantization support @@ -20,9 +20,8 @@ - add explicit detailer steps setting - **SysInfo**: - update to collected data and benchmarks -- **Progress**: +- **Refactor**: - refactored progress monitoring, job updates and live preview -- **Metadata**: - improved metadata save and restore - **Fixes**: - explict clear caches on model load @@ -34,6 +33,7 @@ - samplers test for scale noise before using - scheduler api - controlnet with hires + - controlnet with batch count ## Update for 2024-12-31 diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index b5a0194c2..2ad1df4b8 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -201,7 +201,7 @@ def process_hires(p: processing.StableDiffusionProcessing, output): update_sampler(p, shared.sd_model, second_pass=True) orig_denoise = p.denoising_strength p.denoising_strength = strength - p.task_args.pop('image', None) # remove image override from hires + orig_image = p.task_args.pop('image', None) # remove image override from hires hires_args = set_pipeline_args( p=p, model=shared.sd_model, @@ -245,6 +245,8 @@ def process_hires(p: processing.StableDiffusionProcessing, output): shared.log.error(f'Processing step=hires: args={hires_args} {e}') errors.display(e, 'Processing') modelstats.analyze() + if orig_image is not None: + p.task_args['image'] = orig_image p.denoising_strength = orig_denoise shared.state.job = prev_job shared.state.nextjob() From 08550075f5ba476b2b0d9485f4ae9e7d51941be4 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 6 Jan 2025 09:35:36 -0500 Subject: [PATCH 224/249] add tdd scheduler and lint updates Signed-off-by: Vladimir Mandic --- .pylintrc | 3 + .ruff.toml | 2 + CHANGELOG.md | 7 +- modules/history.py | 10 +- modules/model_stablecascade.py | 2 +- modules/pixelsmith/pixelsmith_pipeline.py | 2 +- modules/schedulers/scheduler_tdd.py | 527 ++++++++++++++++++++++ modules/sd_samplers_diffusers.py | 5 +- modules/timer.py | 2 +- wiki | 2 +- 10 files changed, 551 insertions(+), 11 deletions(-) create mode 100644 modules/schedulers/scheduler_tdd.py diff --git a/.pylintrc b/.pylintrc index 4a2850664..78361f7c4 100644 --- a/.pylintrc +++ b/.pylintrc @@ -26,6 +26,7 @@ ignore-paths=/usr/lib/.*$, modules/omnigen, modules/onnx_impl, modules/pag, + modules/pixelsmith, modules/prompt_parser_xhinker.py, modules/pulid/eva_clip, modules/rife, @@ -36,6 +37,7 @@ ignore-paths=/usr/lib/.*$, modules/unipc, modules/xadapter, repositories, + extensions-builtin/Lora, extensions-builtin/sd-webui-agent-scheduler, extensions-builtin/sd-extension-chainner/nodes, extensions-builtin/sdnext-modernui/node_modules, @@ -150,6 +152,7 @@ disable=abstract-method, consider-using-min-builtin, consider-using-max-builtin, consider-using-sys-exit, + cyclic-import, dangerous-default-value, deprecated-pragma, duplicate-code, diff --git a/.ruff.toml b/.ruff.toml index a2ad0b91a..89bd1586d 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -19,6 +19,7 @@ exclude = [ "modules/meissonic", "modules/omnigen", "modules/pag", + "modules/pixelsmith", "modules/postprocess/aurasr_arch.py", "modules/prompt_parser_xhinker.py", "modules/pulid/eva_clip", @@ -31,6 +32,7 @@ exclude = [ "modules/unipc", "modules/xadapter", "repositories", + "extensions-builtin/Lora", "extensions-builtin/sd-extension-chainner/nodes", "extensions-builtin/sd-webui-agent-scheduler", "extensions-builtin/sdnext-modernui/node_modules", diff --git a/CHANGELOG.md b/CHANGELOG.md index fd44bd9de..a987b5f06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Change Log for SD.Next -## Update for 2025-01-05 +## Update for 2025-01-06 - [Allegro Video](https://huggingface.co/rhymes-ai/Allegro) - optimizations: full offload and quantization support @@ -23,6 +23,11 @@ - **Refactor**: - refactored progress monitoring, job updates and live preview - improved metadata save and restore +- **Schedulers**: + - [TDD](https://github.com/RedAIGC/Target-Driven-Distillation) new super-fast scheduler that can generate images in 4-8 steps + recommended to use with [TDD LoRA](https://huggingface.co/RED-AIGC/TDD/tree/main) +- [Wiki/Docs](https://vladmandic.github.io/sdnext-docs/): + - updated: install, debug, control-howto, zluda - **Fixes**: - explict clear caches on model load - lock adetailer commit: `#a89c01d` diff --git a/modules/history.py b/modules/history.py index f60395edd..3d6f672e8 100644 --- a/modules/history.py +++ b/modules/history.py @@ -47,13 +47,13 @@ def list(self): @property def selected(self): if self.index >= 0 and self.index < self.count: - index = self.index + current_index = self.index self.index = -1 else: - index = 0 - item = self.latents[index] - shared.log.debug(f'History get: index={index} time={item.ts} shape={item.latent.shape} dtype={item.latent.dtype} count={self.count}') - return item.latent.to(devices.device), index + current_index = 0 + item = self.latents[current_index] + shared.log.debug(f'History get: index={current_index} time={item.ts} shape={item.latent.shape} dtype={item.latent.dtype} count={self.count}') + return item.latent.to(devices.device), current_index def find(self, name): for i, item in enumerate(self.latents): diff --git a/modules/model_stablecascade.py b/modules/model_stablecascade.py index 2a7739e55..3c3339dca 100644 --- a/modules/model_stablecascade.py +++ b/modules/model_stablecascade.py @@ -256,7 +256,7 @@ def __call__( if isinstance(self.scheduler, diffusers.DDPMWuerstchenScheduler): timesteps = timesteps[:-1] else: - if hasattr(self.scheduler.config, "clip_sample") and self.scheduler.config.clip_sample: + if hasattr(self.scheduler.config, "clip_sample") and self.scheduler.config.clip_sample: # pylint: disable=no-member self.scheduler.config.clip_sample = False # disample sample clipping # 6. Run denoising loop diff --git a/modules/pixelsmith/pixelsmith_pipeline.py b/modules/pixelsmith/pixelsmith_pipeline.py index 4fd21a2fa..4e04b2d92 100644 --- a/modules/pixelsmith/pixelsmith_pipeline.py +++ b/modules/pixelsmith/pixelsmith_pipeline.py @@ -1683,7 +1683,7 @@ def create_gradient_border(mask, gradient_width=5): mask_first_row = torch.zeros(1, patch_size) mask_first_row[:, ::d_rate] = 1 mask_second_row = torch.roll(mask_first_row, shifts=1, dims=1) - for d in range(1, d_rate): + for _d in range(1, d_rate): stacked_rows = torch.concatenate((mask_first_row, mask_second_row), axis=-2) den_mask = torch.tile(stacked_rows, (patch_size//stacked_rows.shape[0], 1)).to(self.device) den_mask = den_mask[np.newaxis, np.newaxis, ...].to(self.unet.dtype) diff --git a/modules/schedulers/scheduler_tdd.py b/modules/schedulers/scheduler_tdd.py new file mode 100644 index 000000000..03c9da1a5 --- /dev/null +++ b/modules/schedulers/scheduler_tdd.py @@ -0,0 +1,527 @@ +from typing import Union, List, Optional, Tuple +import numpy as np +import torch +from diffusers.utils import deprecate, logging +from diffusers.configuration_utils import register_to_config +from diffusers import DPMSolverSinglestepScheduler +from diffusers.schedulers.scheduling_utils import SchedulerOutput +from diffusers.utils.torch_utils import randn_tensor +# from diffusers.schedulers.scheduling_tcd import * +# from diffusers.schedulers.scheduling_dpmsolver_singlestep import * + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +class TDDScheduler(DPMSolverSinglestepScheduler): + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + beta_start: float = 0.0001, + beta_end: float = 0.02, + beta_schedule: str = "linear", + trained_betas: Optional[np.ndarray] = None, + solver_order: int = 1, + prediction_type: str = "epsilon", + thresholding: bool = False, + dynamic_thresholding_ratio: float = 0.995, + sample_max_value: float = 1.0, + algorithm_type: str = "dpmsolver++", + solver_type: str = "midpoint", + lower_order_final: bool = False, + use_karras_sigmas: Optional[bool] = False, + final_sigmas_type: Optional[str] = "zero", # "zero", "sigma_min" + lambda_min_clipped: float = -float("inf"), + variance_type: Optional[str] = None, + tdd_train_step: int = 250, + special_jump: bool = False, + t_l: int = -1, + use_flow_sigmas: bool = False, + ): + self.t_l = t_l + self.special_jump = special_jump + self.tdd_train_step = tdd_train_step + if algorithm_type == "dpmsolver": + deprecation_message = "algorithm_type `dpmsolver` is deprecated and will be removed in a future version. Choose from `dpmsolver++` or `sde-dpmsolver++` instead" + deprecate("algorithm_types=dpmsolver", "1.0.0", deprecation_message) + + if trained_betas is not None: + self.betas = torch.tensor(trained_betas, dtype=torch.float32) + elif beta_schedule == "linear": + self.betas = torch.linspace(beta_start, beta_end, num_train_timesteps, dtype=torch.float32) + elif beta_schedule == "scaled_linear": + # this schedule is very specific to the latent diffusion model. + self.betas = torch.linspace(beta_start**0.5, beta_end**0.5, num_train_timesteps, dtype=torch.float32) ** 2 + elif beta_schedule == "squaredcos_cap_v2": + # Glide cosine schedule + self.betas = betas_for_alpha_bar(num_train_timesteps) + else: + raise NotImplementedError(f"{beta_schedule} does is not implemented for {self.__class__}") + + self.alphas = 1.0 - self.betas + self.alphas_cumprod = torch.cumprod(self.alphas, dim=0) + # Currently we only support VP-type noise schedule + self.alpha_t = torch.sqrt(self.alphas_cumprod) + self.sigma_t = torch.sqrt(1 - self.alphas_cumprod) + self.lambda_t = torch.log(self.alpha_t) - torch.log(self.sigma_t) + self.sigmas = ((1 - self.alphas_cumprod) / self.alphas_cumprod) ** 0.5 + + # standard deviation of the initial noise distribution + self.init_noise_sigma = 1.0 + + # settings for DPM-Solver + if algorithm_type not in ["dpmsolver", "dpmsolver++"]: + if algorithm_type == "deis": + self.register_to_config(algorithm_type="dpmsolver++") + else: + raise NotImplementedError(f"{algorithm_type} does is not implemented for {self.__class__}") + if solver_type not in ["midpoint", "heun"]: + if solver_type in ["logrho", "bh1", "bh2"]: + self.register_to_config(solver_type="midpoint") + else: + raise NotImplementedError(f"{solver_type} does is not implemented for {self.__class__}") + + if algorithm_type != "dpmsolver++" and final_sigmas_type == "zero": + raise ValueError( + f"`final_sigmas_type` {final_sigmas_type} is not supported for `algorithm_type` {algorithm_type}. Please chooose `sigma_min` instead." + ) + + # setable values + self.num_inference_steps = None + timesteps = np.linspace(0, num_train_timesteps - 1, num_train_timesteps, dtype=np.float32)[::-1].copy() + self.timesteps = torch.from_numpy(timesteps) + self.model_outputs = [None] * solver_order + self.sample = None + self.order_list = self.get_order_list(num_train_timesteps) + self._step_index = None + self._begin_index = None + self.sigmas = self.sigmas.to("cpu") # to avoid too much CPU/GPU communication + + def set_timesteps(self, num_inference_steps: int, device: Union[str, torch.device] = None): + self.num_inference_steps = num_inference_steps + # Clipping the minimum of all lambda(t) for numerical stability. + # This is critical for cosine (squaredcos_cap_v2) noise schedule. + #original_steps = self.config.original_inference_steps + if True: + original_steps=self.tdd_train_step + k = 1000 / original_steps + tcd_origin_timesteps = np.asarray(list(range(1, int(original_steps) + 1))) * k - 1 + else: + tcd_origin_timesteps = np.asarray(list(range(0, int(self.config.num_train_timesteps)))) + # TCD Inference Steps Schedule + tcd_origin_timesteps = tcd_origin_timesteps[::-1].copy() + # Select (approximately) evenly spaced indices from tcd_origin_timesteps. + inference_indices = np.linspace(0, len(tcd_origin_timesteps), num=num_inference_steps, endpoint=False) + inference_indices = np.floor(inference_indices).astype(np.int64) + timesteps = tcd_origin_timesteps[inference_indices] + if self.special_jump: + if self.tdd_train_step == 50: + #timesteps = np.array([999., 879., 759., 499., 259.]) + print(timesteps) + elif self.tdd_train_step == 250: + if num_inference_steps == 5: + timesteps = np.array([999., 875., 751., 499., 251.]) + elif num_inference_steps == 6: + timesteps = np.array([999., 875., 751., 627., 499., 251.]) + elif num_inference_steps == 7: + timesteps = np.array([999., 875., 751., 627., 499., 375., 251.]) + + sigmas = np.array(((1 - self.alphas_cumprod) / self.alphas_cumprod) ** 0.5) + if self.config.use_karras_sigmas: + log_sigmas = np.log(sigmas) + sigmas = np.flip(sigmas).copy() + sigmas = self._convert_to_karras(in_sigmas=sigmas, num_inference_steps=num_inference_steps) + timesteps = np.array([self._sigma_to_t(sigma, log_sigmas) for sigma in sigmas]).round() + else: + sigmas = np.interp(timesteps, np.arange(0, len(sigmas)), sigmas) + + if self.config.final_sigmas_type == "sigma_min": + sigma_last = ((1 - self.alphas_cumprod[0]) / self.alphas_cumprod[0]) ** 0.5 + elif self.config.final_sigmas_type == "zero": + sigma_last = 0 + else: + raise ValueError( + f" `final_sigmas_type` must be one of `sigma_min` or `zero`, but got {self.config.final_sigmas_type}" + ) + sigmas = np.concatenate([sigmas, [sigma_last]]).astype(np.float32) + + self.sigmas = torch.from_numpy(sigmas).to(device=device) + + self.timesteps = torch.from_numpy(timesteps).to(device=device, dtype=torch.int64) + self.model_outputs = [None] * self.config.solver_order + self.sample = None + + if not self.config.lower_order_final and num_inference_steps % self.config.solver_order != 0: + logger.warning( + "Changing scheduler {self.config} to have `lower_order_final` set to True to handle uneven amount of inference steps. Please make sure to always use an even number of `num_inference steps when using `lower_order_final=False`." + ) + self.register_to_config(lower_order_final=True) + + if not self.config.lower_order_final and self.config.final_sigmas_type == "zero": + logger.warning( + " `last_sigmas_type='zero'` is not supported for `lower_order_final=False`. Changing scheduler {self.config} to have `lower_order_final` set to True." + ) + self.register_to_config(lower_order_final=True) + + self.order_list = self.get_order_list(num_inference_steps) + + # add an index counter for schedulers that allow duplicated timesteps + self._step_index = None + self._begin_index = None + self.sigmas = self.sigmas.to("cpu") # to avoid too much CPU/GPU communication + + def set_timesteps_s(self, eta: float = 0.0): + # Clipping the minimum of all lambda(t) for numerical stability. + # This is critical for cosine (squaredcos_cap_v2) noise schedule. + num_inference_steps = self.num_inference_steps + device = self.timesteps.device + if True: + original_steps=self.tdd_train_step + k = 1000 / original_steps + tcd_origin_timesteps = np.asarray(list(range(1, int(original_steps) + 1))) * k - 1 + else: + tcd_origin_timesteps = np.asarray(list(range(0, int(self.config.num_train_timesteps)))) + # TCD Inference Steps Schedule + tcd_origin_timesteps = tcd_origin_timesteps[::-1].copy() + # Select (approximately) evenly spaced indices from tcd_origin_timesteps. + inference_indices = np.linspace(0, len(tcd_origin_timesteps), num=num_inference_steps, endpoint=False) + inference_indices = np.floor(inference_indices).astype(np.int64) + timesteps = tcd_origin_timesteps[inference_indices] + if self.special_jump: + if self.tdd_train_step == 50: + timesteps = np.array([999., 879., 759., 499., 259.]) + elif self.tdd_train_step == 250: + if num_inference_steps == 5: + timesteps = np.array([999., 875., 751., 499., 251.]) + elif num_inference_steps == 6: + timesteps = np.array([999., 875., 751., 627., 499., 251.]) + elif num_inference_steps == 7: + timesteps = np.array([999., 875., 751., 627., 499., 375., 251.]) + + timesteps_s = np.floor((1 - eta) * timesteps).astype(np.int64) + + sigmas_s = np.array(((1 - self.alphas_cumprod) / self.alphas_cumprod) ** 0.5) + if self.config.use_karras_sigmas: + print("have not write") + pass + else: + sigmas_s = np.interp(timesteps_s, np.arange(0, len(sigmas_s)), sigmas_s) + + if self.config.final_sigmas_type == "sigma_min": + sigma_last = ((1 - self.alphas_cumprod[0]) / self.alphas_cumprod[0]) ** 0.5 + elif self.config.final_sigmas_type == "zero": + sigma_last = 0 + else: + raise ValueError( + f" `final_sigmas_type` must be one of `sigma_min` or `zero`, but got {self.config.final_sigmas_type}" + ) + + sigmas_s = np.concatenate([sigmas_s, [sigma_last]]).astype(np.float32) + self.sigmas_s = torch.from_numpy(sigmas_s).to(device=device) + self.timesteps_s = torch.from_numpy(timesteps_s).to(device=device, dtype=torch.int64) + + def step( + self, + model_output: torch.FloatTensor, + timestep: int, + sample: torch.FloatTensor, + eta: float, + generator: Optional[torch.Generator] = None, + return_dict: bool = True, + ) -> Union[SchedulerOutput, Tuple]: + if self.num_inference_steps is None: + raise ValueError( + "Number of inference steps is 'None', you need to run 'set_timesteps' after creating the scheduler" + ) + + if self.step_index is None: + self._init_step_index(timestep) + + if self.step_index == 0: + self.set_timesteps_s(eta) + + model_output = self.convert_model_output(model_output, sample=sample) + for i in range(self.config.solver_order - 1): + self.model_outputs[i] = self.model_outputs[i + 1] + self.model_outputs[-1] = model_output + + order = self.order_list[self.step_index] + + # For img2img denoising might start with order>1 which is not possible + # In this case make sure that the first two steps are both order=1 + while self.model_outputs[-order] is None: + order -= 1 + + # For single-step solvers, we use the initial value at each time with order = 1. + if order == 1: + self.sample = sample + + prev_sample = self.singlestep_dpm_solver_update(self.model_outputs, sample=self.sample, order=order) + + if eta > 0: + if self.step_index != self.num_inference_steps - 1: + + alpha_prod_s = self.alphas_cumprod[self.timesteps_s[self.step_index + 1]] + alpha_prod_t_prev = self.alphas_cumprod[self.timesteps[self.step_index + 1]] + + noise = randn_tensor( + model_output.shape, generator=generator, device=model_output.device, dtype=prev_sample.dtype + ) + prev_sample = (alpha_prod_t_prev / alpha_prod_s).sqrt() * prev_sample + ( + 1 - alpha_prod_t_prev / alpha_prod_s + ).sqrt() * noise + + # upon completion increase step index by one + self._step_index += 1 + + if not return_dict: + return (prev_sample,) + + return SchedulerOutput(prev_sample=prev_sample) + + def dpm_solver_first_order_update( + self, + model_output: torch.FloatTensor, + *args, + sample: torch.FloatTensor = None, + **kwargs, + ) -> torch.FloatTensor: + timestep = args[0] if len(args) > 0 else kwargs.pop("timestep", None) + prev_timestep = args[1] if len(args) > 1 else kwargs.pop("prev_timestep", None) + if sample is None: + if len(args) > 2: + sample = args[2] + else: + raise ValueError(" missing `sample` as a required keyward argument") + if timestep is not None: + deprecate( + "timesteps", + "1.0.0", + "Passing `timesteps` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + if prev_timestep is not None: + deprecate( + "prev_timestep", + "1.0.0", + "Passing `prev_timestep` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + sigma_t, sigma_s = self.sigmas_s[self.step_index + 1], self.sigmas[self.step_index] + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma_t) + alpha_s, sigma_s = self._sigma_to_alpha_sigma_t(sigma_s) + lambda_t = torch.log(alpha_t) - torch.log(sigma_t) + lambda_s = torch.log(alpha_s) - torch.log(sigma_s) + h = lambda_t - lambda_s + if self.config.algorithm_type == "dpmsolver++": + x_t = (sigma_t / sigma_s) * sample - (alpha_t * (torch.exp(-h) - 1.0)) * model_output + elif self.config.algorithm_type == "dpmsolver": + x_t = (alpha_t / alpha_s) * sample - (sigma_t * (torch.exp(h) - 1.0)) * model_output + return x_t + + def singlestep_dpm_solver_second_order_update( + self, + model_output_list: List[torch.FloatTensor], + *args, + sample: torch.FloatTensor = None, + **kwargs, + ) -> torch.FloatTensor: + timestep_list = args[0] if len(args) > 0 else kwargs.pop("timestep_list", None) + prev_timestep = args[1] if len(args) > 1 else kwargs.pop("prev_timestep", None) + if sample is None: + if len(args) > 2: + sample = args[2] + else: + raise ValueError(" missing `sample` as a required keyward argument") + if timestep_list is not None: + deprecate( + "timestep_list", + "1.0.0", + "Passing `timestep_list` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + if prev_timestep is not None: + deprecate( + "prev_timestep", + "1.0.0", + "Passing `prev_timestep` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + sigma_t, sigma_s0, sigma_s1 = ( + self.sigmas_s[self.step_index + 1], + self.sigmas[self.step_index], + self.sigmas[self.step_index - 1], + ) + + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma_t) + alpha_s0, sigma_s0 = self._sigma_to_alpha_sigma_t(sigma_s0) + alpha_s1, sigma_s1 = self._sigma_to_alpha_sigma_t(sigma_s1) + + lambda_t = torch.log(alpha_t) - torch.log(sigma_t) + lambda_s0 = torch.log(alpha_s0) - torch.log(sigma_s0) + lambda_s1 = torch.log(alpha_s1) - torch.log(sigma_s1) + + m0, m1 = model_output_list[-1], model_output_list[-2] + + h, h_0 = lambda_t - lambda_s1, lambda_s0 - lambda_s1 + r0 = h_0 / h + D0, D1 = m1, (1.0 / r0) * (m0 - m1) + if self.config.algorithm_type == "dpmsolver++": + # See https://arxiv.org/abs/2211.01095 for detailed derivations + if self.config.solver_type == "midpoint": + x_t = ( + (sigma_t / sigma_s1) * sample + - (alpha_t * (torch.exp(-h) - 1.0)) * D0 + - 0.5 * (alpha_t * (torch.exp(-h) - 1.0)) * D1 + ) + elif self.config.solver_type == "heun": + x_t = ( + (sigma_t / sigma_s1) * sample + - (alpha_t * (torch.exp(-h) - 1.0)) * D0 + + (alpha_t * ((torch.exp(-h) - 1.0) / h + 1.0)) * D1 + ) + elif self.config.algorithm_type == "dpmsolver": + # See https://arxiv.org/abs/2206.00927 for detailed derivations + if self.config.solver_type == "midpoint": + x_t = ( + (alpha_t / alpha_s1) * sample + - (sigma_t * (torch.exp(h) - 1.0)) * D0 + - 0.5 * (sigma_t * (torch.exp(h) - 1.0)) * D1 + ) + elif self.config.solver_type == "heun": + x_t = ( + (alpha_t / alpha_s1) * sample + - (sigma_t * (torch.exp(h) - 1.0)) * D0 + - (sigma_t * ((torch.exp(h) - 1.0) / h - 1.0)) * D1 + ) + return x_t + + def singlestep_dpm_solver_update( + self, + model_output_list: List[torch.FloatTensor], + *args, + sample: torch.FloatTensor = None, + order: int = None, + **kwargs, + ) -> torch.FloatTensor: + timestep_list = args[0] if len(args) > 0 else kwargs.pop("timestep_list", None) + prev_timestep = args[1] if len(args) > 1 else kwargs.pop("prev_timestep", None) + if sample is None: + if len(args) > 2: + sample = args[2] + else: + raise ValueError(" missing`sample` as a required keyward argument") + if order is None: + if len(args) > 3: + order = args[3] + else: + raise ValueError(" missing `order` as a required keyward argument") + if timestep_list is not None: + deprecate( + "timestep_list", + "1.0.0", + "Passing `timestep_list` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + if prev_timestep is not None: + deprecate( + "prev_timestep", + "1.0.0", + "Passing `prev_timestep` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + if order == 1: + return self.dpm_solver_first_order_update(model_output_list[-1], sample=sample) + elif order == 2: + return self.singlestep_dpm_solver_second_order_update(model_output_list, sample=sample) + else: + raise ValueError(f"Order must be 1, 2, got {order}") + + def convert_model_output( + self, + model_output: torch.FloatTensor, + *args, + sample: torch.FloatTensor = None, + **kwargs, + ) -> torch.FloatTensor: + """ + Convert the model output to the corresponding type the DPMSolver/DPMSolver++ algorithm needs. DPM-Solver is + designed to discretize an integral of the noise prediction model, and DPM-Solver++ is designed to discretize an + integral of the data prediction model. + + + + The algorithm and model type are decoupled. You can use either DPMSolver or DPMSolver++ for both noise + prediction and data prediction models. + + + + Args: + model_output (`torch.FloatTensor`): + The direct output from the learned diffusion model. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + + Returns: + `torch.FloatTensor`: + The converted model output. + """ + timestep = args[0] if len(args) > 0 else kwargs.pop("timestep", None) + if sample is None: + if len(args) > 1: + sample = args[1] + else: + raise ValueError("missing `sample` as a required keyward argument") + if timestep is not None: + deprecate( + "timesteps", + "1.0.0", + "Passing `timesteps` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + # DPM-Solver++ needs to solve an integral of the data prediction model. + if self.config.algorithm_type == "dpmsolver++": + if self.config.prediction_type == "epsilon": + # DPM-Solver and DPM-Solver++ only need the "mean" output. + if self.config.variance_type in ["learned_range"]: + model_output = model_output[:, :3] + sigma = self.sigmas[self.step_index] + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma) + x0_pred = (sample - sigma_t * model_output) / alpha_t + elif self.config.prediction_type == "sample": + x0_pred = model_output + elif self.config.prediction_type == "v_prediction": + sigma = self.sigmas[self.step_index] + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma) + x0_pred = alpha_t * sample - sigma_t * model_output + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample`, or" + " `v_prediction` for the DPMSolverSinglestepScheduler." + ) + + if self.step_index <= self.t_l: + if self.config.thresholding: + x0_pred = self._threshold_sample(x0_pred) + + return x0_pred + # DPM-Solver needs to solve an integral of the noise prediction model. + elif self.config.algorithm_type == "dpmsolver": + if self.config.prediction_type == "epsilon": + # DPM-Solver and DPM-Solver++ only need the "mean" output. + if self.config.variance_type in ["learned_range"]: + model_output = model_output[:, :3] + return model_output + elif self.config.prediction_type == "sample": + sigma = self.sigmas[self.step_index] + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma) + epsilon = (sample - alpha_t * model_output) / sigma_t + return epsilon + elif self.config.prediction_type == "v_prediction": + sigma = self.sigmas[self.step_index] + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma) + epsilon = alpha_t * model_output + sigma_t * sample + return epsilon + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample`, or" + " `v_prediction` for the DPMSolverSinglestepScheduler." + ) diff --git a/modules/sd_samplers_diffusers.py b/modules/sd_samplers_diffusers.py index 6e5f37cd7..9f88bf103 100644 --- a/modules/sd_samplers_diffusers.py +++ b/modules/sd_samplers_diffusers.py @@ -48,6 +48,7 @@ errors.display(e, 'Samplers') try: from modules.schedulers.scheduler_tcd import TCDScheduler # pylint: disable=ungrouped-imports + from modules.schedulers.scheduler_tdd import TDDScheduler # pylint: disable=ungrouped-imports from modules.schedulers.scheduler_dc import DCSolverMultistepScheduler # pylint: disable=ungrouped-imports from modules.schedulers.scheduler_vdm import VDMScheduler # pylint: disable=ungrouped-imports from modules.schedulers.scheduler_dpm_flowmatch import FlowMatchDPMSolverMultistepScheduler # pylint: disable=ungrouped-imports @@ -98,7 +99,8 @@ 'VDM Solver': { 'clip_sample_range': 2.0, }, 'LCM': { 'beta_start': 0.00085, 'beta_end': 0.012, 'beta_schedule': "scaled_linear", 'set_alpha_to_one': True, 'rescale_betas_zero_snr': False, 'thresholding': False, 'timestep_spacing': 'linspace' }, 'TCD': { 'set_alpha_to_one': True, 'rescale_betas_zero_snr': False, 'beta_schedule': 'scaled_linear' }, - 'UFOGen': {}, + 'TDD': { }, + 'UFOGen': { }, 'BDIA DDIM': { 'clip_sample': False, 'set_alpha_to_one': True, 'steps_offset': 0, 'clip_sample_range': 1.0, 'sample_max_value': 1.0, 'timestep_spacing': 'leading', 'rescale_betas_zero_snr': False, 'thresholding': False, 'gamma': 1.0 }, 'PNDM': { 'skip_prk_steps': False, 'set_alpha_to_one': False, 'steps_offset': 0, 'timestep_spacing': 'linspace' }, @@ -157,6 +159,7 @@ SamplerData('LCM', lambda model: DiffusionSampler('LCM', LCMScheduler, model), [], {}), SamplerData('TCD', lambda model: DiffusionSampler('TCD', TCDScheduler, model), [], {}), + SamplerData('TDD', lambda model: DiffusionSampler('TDD', TDDScheduler, model), [], {}), SamplerData('UFOGen', lambda model: DiffusionSampler('UFOGen', UFOGenScheduler, model), [], {}), SamplerData('Same as primary', None, [], {}), diff --git a/modules/timer.py b/modules/timer.py index 977206e06..3c7d42f6d 100644 --- a/modules/timer.py +++ b/modules/timer.py @@ -53,7 +53,7 @@ def dct(self, min_time=default_min_time): if self.profile: res = {k: round(v, 4) for k, v in self.records.items()} res = {k: round(v, 2) for k, v in self.records.items() if v >= min_time} - res = {k: v for k, v in sorted(res.items(), key=lambda x: x[1], reverse=True)} # noqa: C416 + res = {k: v for k, v in sorted(res.items(), key=lambda x: x[1], reverse=True)} # noqa: C416 # pylint: disable=unnecessary-comprehension return res def reset(self): diff --git a/wiki b/wiki index 1f792d308..a0c9483d8 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 1f792d30858b511ef728531250f3c1a283e4c211 +Subproject commit a0c9483d8a1a979911b056f86afa598fca635857 From bef14786d57ee384eaef72048f155b2368f1e682 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Mon, 6 Jan 2025 09:49:07 -0500 Subject: [PATCH 225/249] update diffusers and register mask_processor Signed-off-by: Vladimir Mandic --- installer.py | 8 +++++--- modules/sd_models.py | 3 +++ requirements.txt | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/installer.py b/installer.py index 11e7f1d94..8d6365462 100644 --- a/installer.py +++ b/installer.py @@ -459,13 +459,15 @@ def check_python(supported_minors=[9, 10, 11, 12], reason=None): def check_diffusers(): if args.skip_all or args.skip_git: return - sha = '6dfaec348780c6153a4cfd03a01972a291d67f82' # diffusers commit hash + sha = 'b5726358cf125f2fa1a596dce321e91a225a57e4' # diffusers commit hash pkg = pkg_resources.working_set.by_key.get('diffusers', None) minor = int(pkg.version.split('.')[1] if pkg is not None else 0) cur = opts.get('diffusers_version', '') if minor > 0 else '' if (minor == 0) or (cur != sha): - log.info(f'Diffusers {"install" if minor == 0 else "upgrade"}: package={pkg} current={cur} target={sha}') - if minor > 0: + if minor == 0: + log.info(f'Diffusers install: commit={sha}') + else: + log.info(f'Diffusers update: package={pkg} current={cur} target={sha}') pip('uninstall --yes diffusers', ignore=True, quiet=True, uv=False) pip(f'install --upgrade git+https://github.com/huggingface/diffusers@{sha}', ignore=False, quiet=True, uv=False) global diffusers_commit # pylint: disable=global-statement diff --git a/modules/sd_models.py b/modules/sd_models.py index d9e07e1de..8b0eb0a89 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -1242,6 +1242,7 @@ def set_diffuser_pipe(pipe, new_pipe_type): default_scheduler = getattr(pipe, "default_scheduler", None) image_encoder = getattr(pipe, "image_encoder", None) feature_extractor = getattr(pipe, "feature_extractor", None) + mask_processor = getattr(pipe, "mask_processor", None) if new_pipe is None: if hasattr(pipe, 'config'): # real pipeline which can be auto-switched @@ -1289,6 +1290,8 @@ def set_diffuser_pipe(pipe, new_pipe_type): new_pipe.image_encoder = image_encoder if feature_extractor is not None: new_pipe.feature_extractor = feature_extractor + if mask_processor is not None: + new_pipe.mask_processor = mask_processor if new_pipe.__class__.__name__ in ['FluxPipeline', 'StableDiffusion3Pipeline']: new_pipe.register_modules(image_encoder = image_encoder) new_pipe.register_modules(feature_extractor = feature_extractor) diff --git a/requirements.txt b/requirements.txt index 9b2a52ffa..268082699 100644 --- a/requirements.txt +++ b/requirements.txt @@ -32,7 +32,7 @@ invisible-watermark pi-heif # versioned -safetensors==0.4.5 +safetensors==0.5.0 tensordict==0.1.2 peft==0.14.0 httpx==0.24.1 From 90ff61439672ec001371de063c79ebeb083dc7c5 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 7 Jan 2025 10:12:47 -0500 Subject: [PATCH 226/249] refactor detailer Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 9 ++++++++- modules/control/run.py | 10 ++++++++-- modules/img2img.py | 16 ++++++++++----- modules/postprocess/yolo.py | 24 +++++++++++----------- modules/processing.py | 6 +++--- modules/processing_args.py | 2 +- modules/processing_class.py | 13 ++++++++++-- modules/processing_diffusers.py | 2 +- modules/processing_helpers.py | 2 +- modules/processing_info.py | 8 ++++++-- modules/processing_original.py | 6 +++--- modules/shared.py | 2 -- modules/txt2img.py | 11 ++++++++--- modules/ui_control.py | 16 ++++++++++----- modules/ui_img2img.py | 16 ++++++++++----- modules/ui_sections.py | 15 -------------- modules/ui_txt2img.py | 16 ++++++++++----- scripts/xyz_grid_classes.py | 7 +++++-- scripts/xyz_grid_on.py | 6 +++--- scripts/xyz_grid_shared.py | 35 +++++++++++++++++++++++++-------- wiki | 2 +- 21 files changed, 142 insertions(+), 82 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a987b5f06..c5d707b33 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,8 +26,15 @@ - **Schedulers**: - [TDD](https://github.com/RedAIGC/Target-Driven-Distillation) new super-fast scheduler that can generate images in 4-8 steps recommended to use with [TDD LoRA](https://huggingface.co/RED-AIGC/TDD/tree/main) +- **Detailer**: + - add explicit detailer prompt and negative prompt + - move steps, strength, prompt, negative from settings into ui params + - set/restore detailer metadata + - new [detailer wiki](https://github.com/vladmandic/automatic/wiki/Detailer) +- **XYZ Grid** + - add prompt search&replace options: *primary, refine, detailer, all* - [Wiki/Docs](https://vladmandic.github.io/sdnext-docs/): - - updated: install, debug, control-howto, zluda + - updated: Detailer, Install, Debug, Control-HowTo, ZLUDA - **Fixes**: - explict clear caches on model load - lock adetailer commit: `#a89c01d` diff --git a/modules/control/run.py b/modules/control/run.py index 8cecb93af..b96718e0d 100644 --- a/modules/control/run.py +++ b/modules/control/run.py @@ -222,7 +222,8 @@ def control_run(state: str = '', steps: int = 20, sampler_index: int = None, seed: int = -1, subseed: int = -1, subseed_strength: float = 0, seed_resize_from_h: int = -1, seed_resize_from_w: int = -1, cfg_scale: float = 6.0, clip_skip: float = 1.0, image_cfg_scale: float = 6.0, diffusers_guidance_rescale: float = 0.7, pag_scale: float = 0.0, pag_adaptive: float = 0.5, cfg_end: float = 1.0, - full_quality: bool = True, detailer: bool = False, tiling: bool = False, hidiffusion: bool = False, + full_quality: bool = True, tiling: bool = False, hidiffusion: bool = False, + detailer_enabled: bool = True, detailer_prompt: str = '', detailer_negative: str = '', detailer_steps: int = 10, detailer_strength: float = 0.3, hdr_mode: int = 0, hdr_brightness: float = 0, hdr_color: float = 0, hdr_sharpen: float = 0, hdr_clamp: bool = False, hdr_boundary: float = 4.0, hdr_threshold: float = 0.95, hdr_maximize: bool = False, hdr_max_center: float = 0.6, hdr_max_boundry: float = 1.0, hdr_color_picker: str = None, hdr_tint_ratio: float = 0, resize_mode_before: int = 0, resize_name_before: str = 'None', resize_context_before: str = 'None', width_before: int = 512, height_before: int = 512, scale_by_before: float = 1.0, selected_scale_tab_before: int = 0, @@ -286,9 +287,14 @@ def control_run(state: str = '', pag_scale = pag_scale, pag_adaptive = pag_adaptive, full_quality = full_quality, - detailer = detailer, tiling = tiling, hidiffusion = hidiffusion, + # detailer + detailer_enabled = detailer_enabled, + detailer_prompt = detailer_prompt, + detailer_negative = detailer_negative, + detailer_steps = detailer_steps, + detailer_strength = detailer_strength, # resize resize_mode = resize_mode_before if resize_name_before != 'None' else 0, resize_name = resize_name_before, diff --git a/modules/img2img.py b/modules/img2img.py index 2e3eca54d..7a5a33cd0 100644 --- a/modules/img2img.py +++ b/modules/img2img.py @@ -139,7 +139,8 @@ def img2img(id_task: str, state: str, mode: int, sampler_index, mask_blur, mask_alpha, inpainting_fill, - full_quality, detailer, tiling, hidiffusion, + full_quality, tiling, hidiffusion, + detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, n_iter, batch_size, cfg_scale, image_cfg_scale, diffusers_guidance_rescale, @@ -158,14 +159,15 @@ def img2img(id_task: str, state: str, mode: int, hdr_mode, hdr_brightness, hdr_color, hdr_sharpen, hdr_clamp, hdr_boundary, hdr_threshold, hdr_maximize, hdr_max_center, hdr_max_boundry, hdr_color_picker, hdr_tint_ratio, enable_hr, hr_sampler_index, hr_denoising_strength, hr_resize_mode, hr_resize_context, hr_upscaler, hr_force, hr_second_pass_steps, hr_scale, hr_resize_x, hr_resize_y, refiner_steps, hr_refiner_start, refiner_prompt, refiner_negative, override_settings_texts, - *args): # pylint: disable=unused-argument + *args): + + + debug(f'img2img: {id_task}') if shared.sd_model is None: shared.log.warning('Aborted: op=img model not loaded') return [], '', '', 'Error: model not loaded' - debug(f'img2img: id_task={id_task}|mode={mode}|prompt={prompt}|negative_prompt={negative_prompt}|prompt_styles={prompt_styles}|init_img={init_img}|sketch={sketch}|init_img_with_mask={init_img_with_mask}|inpaint_color_sketch={inpaint_color_sketch}|inpaint_color_sketch_orig={inpaint_color_sketch_orig}|init_img_inpaint={init_img_inpaint}|init_mask_inpaint={init_mask_inpaint}|steps={steps}|sampler_index={sampler_index}||mask_blur={mask_blur}|mask_alpha={mask_alpha}|inpainting_fill={inpainting_fill}|full_quality={full_quality}|detailer={detailer}|tiling={tiling}|hidiffusion={hidiffusion}|n_iter={n_iter}|batch_size={batch_size}|cfg_scale={cfg_scale}|image_cfg_scale={image_cfg_scale}|clip_skip={clip_skip}|denoising_strength={denoising_strength}|seed={seed}|subseed{subseed}|subseed_strength={subseed_strength}|seed_resize_from_h={seed_resize_from_h}|seed_resize_from_w={seed_resize_from_w}|selected_scale_tab={selected_scale_tab}|height={height}|width={width}|scale_by={scale_by}|resize_mode={resize_mode}|resize_name={resize_name}|resize_context={resize_context}|inpaint_full_res={inpaint_full_res}|inpaint_full_res_padding={inpaint_full_res_padding}|inpainting_mask_invert={inpainting_mask_invert}|img2img_batch_files={img2img_batch_files}|img2img_batch_input_dir={img2img_batch_input_dir}|img2img_batch_output_dir={img2img_batch_output_dir}|img2img_batch_inpaint_mask_dir={img2img_batch_inpaint_mask_dir}|override_settings_texts={override_settings_texts}') - if sampler_index is None: shared.log.warning('Sampler: invalid') sampler_index = 0 @@ -240,9 +242,13 @@ def img2img(id_task: str, state: str, mode: int, width=width, height=height, full_quality=full_quality, - detailer=detailer, tiling=tiling, hidiffusion=hidiffusion, + detailer_enabled=detailer_enabled, + detailer_prompt=detailer_prompt, + detailer_negative=detailer_negative, + detailer_steps=detailer_steps, + detailer_strength=detailer_strength, init_images=[image], mask=mask, mask_blur=mask_blur, diff --git a/modules/postprocess/yolo.py b/modules/postprocess/yolo.py index 4011147ec..44016c18b 100644 --- a/modules/postprocess/yolo.py +++ b/modules/postprocess/yolo.py @@ -207,8 +207,8 @@ def restore(self, np_image, p: processing.StableDiffusionProcessing = None): resolution = 512 if shared.sd_model_type in ['none', 'sd', 'lcm', 'unknown'] else 1024 orig_prompt: str = orig_p.get('all_prompts', [''])[0] orig_negative: str = orig_p.get('all_negative_prompts', [''])[0] - prompt: str = orig_p.get('refiner_prompt', '') - negative: str = orig_p.get('refiner_negative', '') + prompt: str = orig_p.get('detailer_prompt', '') + negative: str = orig_p.get('detailer_negative', '') if len(prompt) == 0: prompt = orig_prompt else: @@ -230,9 +230,9 @@ def restore(self, np_image, p: processing.StableDiffusionProcessing = None): 'n_iter': 1, 'prompt': prompt, 'negative_prompt': negative, - 'denoising_strength': shared.opts.detailer_strength, + 'denoising_strength': p.detailer_strength, 'sampler_name': orig_p.get('hr_sampler_name', 'default'), - 'steps': shared.opts.detailer_steps, + 'steps': p.detailer_steps, 'styles': [], 'inpaint_full_res': True, 'inpainting_mask_invert': 0, @@ -309,7 +309,6 @@ def ui(self, tab: str): def ui_settings_change(detailers, classes, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou, steps): shared.opts.detailer_models = detailers shared.opts.detailer_classes = classes - shared.opts.detailer_strength = strength shared.opts.detailer_padding = padding shared.opts.detailer_blur = blur shared.opts.detailer_conf = min_confidence @@ -317,9 +316,8 @@ def ui_settings_change(detailers, classes, strength, padding, blur, min_confiden shared.opts.detailer_min_size = min_size shared.opts.detailer_max_size = max_size shared.opts.detailer_iou = iou - shared.opts.detailer_steps = steps shared.opts.save(shared.config_filename, silent=True) - shared.log.debug(f'Detailer settings: models={shared.opts.detailer_models} classes={shared.opts.detailer_classes} strength={shared.opts.detailer_strength} conf={shared.opts.detailer_conf} max={shared.opts.detailer_max} iou={shared.opts.detailer_iou} size={shared.opts.detailer_min_size}-{shared.opts.detailer_max_size} padding={shared.opts.detailer_padding} steps={shared.opts.detailer_steps}') + shared.log.debug(f'Detailer settings: models={detailers} classes={classes} strength={strength} conf={min_confidence} max={max_detected} iou={iou} size={min_size}-{max_size} padding={padding} steps={steps}') with gr.Accordion(open=False, label="Detailer", elem_id=f"{tab}_detailer_accordion", elem_classes=["small-accordion"], visible=shared.native): with gr.Row(): @@ -330,8 +328,12 @@ def ui_settings_change(detailers, classes, strength, padding, blur, min_confiden with gr.Row(): classes = gr.Textbox(label="Classes", placeholder="Classes", elem_id=f"{tab}_detailer_classes") with gr.Row(): - steps = gr.Slider(label="Detailer steps", elem_id=f"{tab}_detailer_steps", value=shared.opts.detailer_steps, min=0, max=99, step=1) - strength = gr.Slider(label="Detailer strength", elem_id=f"{tab}_detailer_strength", value=shared.opts.detailer_strength, minimum=0, maximum=1, step=0.01) + prompt = gr.Textbox(label="Detailer prompt", value='', placeholder='Detailer prompt', lines=2) + with gr.Row(): + negative = gr.Textbox(label="Detailer negative prompt", value='', placeholder='Detailer negative prompt', lines=2) + with gr.Row(): + steps = gr.Slider(label="Detailer steps", elem_id=f"{tab}_detailer_steps", value=10, min=0, max=99, step=1) + strength = gr.Slider(label="Detailer strength", elem_id=f"{tab}_detailer_strength", value=0.3, minimum=0, maximum=1, step=0.01) with gr.Row(): max_detected = gr.Slider(label="Max detected", elem_id=f"{tab}_detailer_max", value=shared.opts.detailer_max, min=1, maximum=10, step=1) with gr.Row(): @@ -347,7 +349,6 @@ def ui_settings_change(detailers, classes, strength, padding, blur, min_confiden max_size = gr.Slider(label="Max size", elem_id=f"{tab}_detailer_max_size", value=max_size, minimum=0.0, maximum=1.0, step=0.05) detailers.change(fn=ui_settings_change, inputs=[detailers, classes, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou, steps], outputs=[]) classes.change(fn=ui_settings_change, inputs=[detailers, classes, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou, steps], outputs=[]) - strength.change(fn=ui_settings_change, inputs=[detailers, classes, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou, steps], outputs=[]) padding.change(fn=ui_settings_change, inputs=[detailers, classes, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou, steps], outputs=[]) blur.change(fn=ui_settings_change, inputs=[detailers, classes, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou, steps], outputs=[]) min_confidence.change(fn=ui_settings_change, inputs=[detailers, classes, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou, steps], outputs=[]) @@ -355,8 +356,7 @@ def ui_settings_change(detailers, classes, strength, padding, blur, min_confiden min_size.change(fn=ui_settings_change, inputs=[detailers, classes, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou, steps], outputs=[]) max_size.change(fn=ui_settings_change, inputs=[detailers, classes, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou, steps], outputs=[]) iou.change(fn=ui_settings_change, inputs=[detailers, classes, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou, steps], outputs=[]) - steps.change(fn=ui_settings_change, inputs=[detailers, classes, strength, padding, blur, min_confidence, max_detected, min_size, max_size, iou, steps], outputs=[]) - return enabled + return enabled, prompt, negative, steps, strength def initialize(): diff --git a/modules/processing.py b/modules/processing.py index c2221823e..99fb9f7f3 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -53,8 +53,8 @@ def __init__(self, p: StableDiffusionProcessing, images_list, seed=-1, info=None self.batch_size = max(1, p.batch_size) self.restore_faces = p.restore_faces or False self.face_restoration_model = shared.opts.face_restoration_model if p.restore_faces else None - self.detailer = p.detailer or False - self.detailer_model = shared.opts.detailer_model if p.detailer else None + self.detailer = p.detailer_enabled or False + self.detailer_model = shared.opts.detailer_model if p.detailer_enabled else None self.sd_model_hash = getattr(shared.sd_model, 'sd_model_hash', '') if model_data.sd_model is not None else '' self.seed_resize_from_w = p.seed_resize_from_w self.seed_resize_from_h = p.seed_resize_from_h @@ -374,7 +374,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: sample = face_restoration.restore_faces(sample, p) if sample is not None: image = Image.fromarray(sample) - if p.detailer: + if p.detailer_enabled: p.ops.append('detailer') if not p.do_not_save_samples and shared.opts.save_images_before_detailer: info = create_infotext(p, p.prompts, p.seeds, p.subseeds, index=i) diff --git a/modules/processing_args.py b/modules/processing_args.py index 7b27e8c31..2bc1c1ce2 100644 --- a/modules/processing_args.py +++ b/modules/processing_args.py @@ -65,7 +65,7 @@ def task_specific_kwargs(p, model): elif (sd_models.get_diffusers_task(model) == sd_models.DiffusersTaskType.INPAINTING or is_img2img_model) and len(getattr(p, 'init_images', [])) > 0: if shared.sd_model_type == 'sdxl' and hasattr(model, 'register_to_config'): model.register_to_config(requires_aesthetics_score = False) - if p.detailer: + if p.detailer_enabled: p.ops.append('detailer') else: p.ops.append('inpaint') diff --git a/modules/processing_class.py b/modules/processing_class.py index 60aa52c50..3eeb73622 100644 --- a/modules/processing_class.py +++ b/modules/processing_class.py @@ -52,8 +52,13 @@ def __init__(self, # other hidiffusion: bool = False, do_not_reload_embeddings: bool = False, - detailer: bool = False, restore_faces: bool = False, + # detailer + detailer_enabled: bool = False, + detailer_prompt: str = '', + detailer_negative: str = '', + detailer_steps: int = 10, + detailer_strength: float = 0.3, # hdr corrections hdr_mode: int = 0, hdr_brightness: float = 0, @@ -167,7 +172,11 @@ def __init__(self, self.full_quality = full_quality self.hidiffusion = hidiffusion self.do_not_reload_embeddings = do_not_reload_embeddings - self.detailer = detailer + self.detailer_enabled = detailer_enabled + self.detailer_prompt = detailer_prompt + self.detailer_negative = detailer_negative + self.detailer_steps = detailer_steps + self.detailer_strength = detailer_strength self.restore_faces = restore_faces self.init_images = init_images self.resize_mode = resize_mode diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index 2ad1df4b8..a21712fae 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -47,7 +47,7 @@ def restore_state(p: processing.StableDiffusionProcessing): p.init_images = None if state == 'reprocess_detail': p.skip = ['encode', 'base', 'hires'] - p.detailer = True + p.detailer_enabled = True shared.log.info(f'Restore state: op={p.state} skip={p.skip}') return p diff --git a/modules/processing_helpers.py b/modules/processing_helpers.py index fe6fa3b3c..e4a35442b 100644 --- a/modules/processing_helpers.py +++ b/modules/processing_helpers.py @@ -601,7 +601,7 @@ def get_job_name(p, model): else: return 'Image' elif sd_models.get_diffusers_task(model) == sd_models.DiffusersTaskType.INPAINTING: - if p.detailer: + if p.detailer_enabled: return 'Detailer' else: return 'Inpaint' diff --git a/modules/processing_info.py b/modules/processing_info.py index 54bfe331d..f3c8eea81 100644 --- a/modules/processing_info.py +++ b/modules/processing_info.py @@ -52,8 +52,8 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts=None, all_seeds=No "Model": None if (not shared.opts.add_model_name_to_info) or (not shared.sd_model.sd_checkpoint_info.model_name) else shared.sd_model.sd_checkpoint_info.model_name.replace(',', '').replace(':', ''), "Model hash": getattr(p, 'sd_model_hash', None if (not shared.opts.add_model_hash_to_info) or (not shared.sd_model.sd_model_hash) else shared.sd_model.sd_model_hash), "VAE": (None if not shared.opts.add_model_name_to_info or sd_vae.loaded_vae_file is None else os.path.splitext(os.path.basename(sd_vae.loaded_vae_file))[0]) if p.full_quality else 'TAESD', - "Prompt2": p.refiner_prompt if len(p.refiner_prompt) > 0 else None, - "Negative2": p.refiner_negative if len(p.refiner_negative) > 0 else None, + "Refiner prompt": p.refiner_prompt if len(p.refiner_prompt) > 0 else None, + "Refiner negative": p.refiner_negative if len(p.refiner_negative) > 0 else None, "Styles": "; ".join(p.styles) if p.styles is not None and len(p.styles) > 0 else None, # sdnext "App": 'SD.Next', @@ -136,6 +136,10 @@ def create_infotext(p: StableDiffusionProcessing, all_prompts=None, all_seeds=No args['Size name mask'] = p.resize_name_mask if 'detailer' in p.ops: args["Detailer"] = ', '.join(shared.opts.detailer_models) + args["Detailer steps"] = p.detailer_steps + args["Detailer strength"] = p.detailer_strength + args["Detailer prompt"] = p.detailer_prompt if len(p.detailer_prompt) > 0 else None + args["Detailer negative"] = p.detailer_negative if len(p.detailer_negative) > 0 else None if 'color' in p.ops: args["Color correction"] = True # embeddings diff --git a/modules/processing_original.py b/modules/processing_original.py index 649023aae..5ea255d72 100644 --- a/modules/processing_original.py +++ b/modules/processing_original.py @@ -90,11 +90,11 @@ def sample_txt2img(p: processing.StableDiffusionProcessingTxt2Img, conditioning, for i, x_sample in enumerate(decoded_samples): x_sample = validate_sample(x_sample) image = Image.fromarray(x_sample) - bak_extra_generation_params, bak_detailer = p.extra_generation_params, p.detailer + orig_extra_generation_params, orig_detailer = p.extra_generation_params, p.detailer_denabled p.extra_generation_params = {} - p.detailer = False + p.detailer_denabled = False info = processing.create_infotext(p, p.all_prompts, p.all_seeds, p.all_subseeds, [], iteration=p.iteration, position_in_batch=i) - p.extra_generation_params, p.detailer = bak_extra_generation_params, bak_detailer + p.extra_generation_params, p.detailer_enabled = orig_extra_generation_params, orig_detailer images.save_image(image, p.outpath_samples, "", seeds[i], prompts[i], shared.opts.samples_format, info=info, suffix="-before-hires") if latent_scale_mode is None or p.hr_force: # non-latent upscaling shared.state.job = 'Upscale' diff --git a/modules/shared.py b/modules/shared.py index fd05f2e44..238878094 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -865,8 +865,6 @@ def get_default_modes(): "detailer_max_size": OptionInfo(1.0, "Max object size", gr.Slider, {"minimum": 0.1, "maximum": 1, "step": 0.05, "visible": False}), "detailer_padding": OptionInfo(20, "Item padding", gr.Slider, {"minimum": 0, "maximum": 100, "step": 1, "visible": False}), "detailer_blur": OptionInfo(10, "Item edge blur", gr.Slider, {"minimum": 0, "maximum": 100, "step": 1, "visible": False}), - "detailer_steps": OptionInfo(10, "Detailer steps", gr.Slider, {"minimum": 0, "maximum": 99, "step": 1, "visible": False}), - "detailer_strength": OptionInfo(0.5, "Detailer strength", gr.Slider, {"minimum": 0, "maximum": 1, "step": 0.01, "visible": False}), "detailer_models": OptionInfo(['face-yolo8n'], "Detailer models", gr.Dropdown, lambda: {"multiselect":True, "choices": list(yolo.list), "visible": False}), "detailer_unload": OptionInfo(False, "Move detailer model to CPU when complete"), "detailer_augment": OptionInfo(True, "Detailer use model augment"), diff --git a/modules/txt2img.py b/modules/txt2img.py index e82c744a2..e2cf7af77 100644 --- a/modules/txt2img.py +++ b/modules/txt2img.py @@ -11,7 +11,8 @@ def txt2img(id_task, state, prompt, negative_prompt, prompt_styles, steps, sampler_index, hr_sampler_index, - full_quality, detailer, tiling, hidiffusion, + full_quality, tiling, hidiffusion, + detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, n_iter, batch_size, cfg_scale, image_cfg_scale, diffusers_guidance_rescale, pag_scale, pag_adaptive, cfg_end, clip_skip, @@ -24,7 +25,7 @@ def txt2img(id_task, state, override_settings_texts, *args): - debug(f'txt2img: id_task={id_task}|prompt={prompt}|negative={negative_prompt}|styles={prompt_styles}|steps={steps}|sampler_index={sampler_index}|hr_sampler_index={hr_sampler_index}|full_quality={full_quality}|detailer={detailer}|tiling={tiling}|hidiffusion={hidiffusion}|batch_count={n_iter}|batch_size={batch_size}|cfg_scale={cfg_scale}|clip_skip={clip_skip}|seed={seed}|subseed={subseed}|subseed_strength={subseed_strength}|seed_resize_from_h={seed_resize_from_h}|seed_resize_from_w={seed_resize_from_w}|height={height}|width={width}|enable_hr={enable_hr}|denoising_strength={denoising_strength}|hr_resize_mode={hr_resize_mode}|hr_resize_context={hr_resize_context}|hr_scale={hr_scale}|hr_upscaler={hr_upscaler}|hr_force={hr_force}|hr_second_pass_steps={hr_second_pass_steps}|hr_resize_x={hr_resize_x}|hr_resize_y={hr_resize_y}|image_cfg_scale={image_cfg_scale}|diffusers_guidance_rescale={diffusers_guidance_rescale}|refiner_steps={refiner_steps}|refiner_start={refiner_start}|refiner_prompt={refiner_prompt}|refiner_negative={refiner_negative}|override_settings={override_settings_texts}') + debug(f'txt2img: {id_task}') if shared.sd_model is None: shared.log.warning('Aborted: op=txt model not loaded') @@ -64,7 +65,11 @@ def txt2img(id_task, state, width=width, height=height, full_quality=full_quality, - detailer=detailer, + detailer_enabled=detailer_enabled, + detailer_prompt=detailer_prompt, + detailer_negative=detailer_negative, + detailer_steps=detailer_steps, + detailer_strength=detailer_strength, tiling=tiling, hidiffusion=hidiffusion, enable_hr=enable_hr, diff --git a/modules/ui_control.py b/modules/ui_control.py index a18d2305c..952b956f8 100644 --- a/modules/ui_control.py +++ b/modules/ui_control.py @@ -177,7 +177,7 @@ def create_ui(_blocks: gr.Blocks=None): video_type.change(fn=helpers.video_type_change, inputs=[video_type], outputs=[video_duration, video_loop, video_pad, video_interpolate]) enable_hr, hr_sampler_index, hr_denoising_strength, hr_resize_mode, hr_resize_context, hr_upscaler, hr_force, hr_second_pass_steps, hr_scale, hr_resize_x, hr_resize_y, refiner_steps, refiner_start, refiner_prompt, refiner_negative = ui_sections.create_hires_inputs('control') - detailer = shared.yolo.ui('control') + detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength = shared.yolo.ui('control') with gr.Row(): override_settings = ui_common.create_override_inputs('control') @@ -567,7 +567,8 @@ def create_ui(_blocks: gr.Blocks=None): prompt, negative, styles, steps, sampler_index, seed, subseed, subseed_strength, seed_resize_from_h, seed_resize_from_w, - cfg_scale, clip_skip, image_cfg_scale, guidance_rescale, pag_scale, pag_adaptive, cfg_end, full_quality, detailer, tiling, hidiffusion, + cfg_scale, clip_skip, image_cfg_scale, guidance_rescale, pag_scale, pag_adaptive, cfg_end, full_quality, tiling, hidiffusion, + detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, hdr_mode, hdr_brightness, hdr_color, hdr_sharpen, hdr_clamp, hdr_boundary, hdr_threshold, hdr_maximize, hdr_max_center, hdr_max_boundry, hdr_color_picker, hdr_tint_ratio, resize_mode_before, resize_name_before, resize_context_before, width_before, height_before, scale_by_before, selected_scale_tab_before, resize_mode_after, resize_name_after, resize_context_after, width_after, height_after, scale_by_after, selected_scale_tab_after, @@ -652,9 +653,14 @@ def create_ui(_blocks: gr.Blocks=None): (image_cfg_scale, "Hires CFG scale"), (guidance_rescale, "CFG rescale"), (full_quality, "Full quality"), - (detailer, "Detailer"), (tiling, "Tiling"), (hidiffusion, "HiDiffusion"), + # detailer + (detailer_enabled, "Detailer"), + (detailer_prompt, "Detailer prompt"), + (detailer_negative, "Detailer negative"), + (detailer_steps, "Detailer steps"), + (detailer_strength, "Detailer strength"), # second pass (enable_hr, "Second pass"), (enable_hr, "Refine"), @@ -673,8 +679,8 @@ def create_ui(_blocks: gr.Blocks=None): # refiner (refiner_start, "Refiner start"), (refiner_steps, "Refiner steps"), - (refiner_prompt, "Prompt2"), - (refiner_negative, "Negative2"), + (refiner_prompt, "refiner prompt"), + (refiner_negative, "Refiner negative"), # pag (pag_scale, "PAG scale"), (pag_adaptive, "PAG adaptive"), diff --git a/modules/ui_img2img.py b/modules/ui_img2img.py index 74bfd0519..0b59e1b09 100644 --- a/modules/ui_img2img.py +++ b/modules/ui_img2img.py @@ -133,7 +133,7 @@ def fn_img_composite_change(img, img_composite): full_quality, tiling, hidiffusion, cfg_scale, clip_skip, image_cfg_scale, diffusers_guidance_rescale, pag_scale, pag_adaptive, cfg_end = ui_sections.create_advanced_inputs('img2img') hdr_mode, hdr_brightness, hdr_color, hdr_sharpen, hdr_clamp, hdr_boundary, hdr_threshold, hdr_maximize, hdr_max_center, hdr_max_boundry, hdr_color_picker, hdr_tint_ratio = ui_sections.create_correction_inputs('img2img') enable_hr, hr_sampler_index, hr_denoising_strength, hr_resize_mode, hr_resize_context, hr_upscaler, hr_force, hr_second_pass_steps, hr_scale, hr_resize_x, hr_resize_y, refiner_steps, hr_refiner_start, refiner_prompt, refiner_negative = ui_sections.create_hires_inputs('txt2img') - detailer = shared.yolo.ui('img2img') + detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength = shared.yolo.ui('img2img') # with gr.Group(elem_id="inpaint_controls", visible=False) as inpaint_controls: with gr.Accordion(open=False, label="Mask", elem_classes=["small-accordion"], elem_id="img2img_mask_group") as inpaint_controls: @@ -174,7 +174,8 @@ def select_img2img_tab(tab): sampler_index, mask_blur, mask_alpha, inpainting_fill, - full_quality, detailer, tiling, hidiffusion, + full_quality, tiling, hidiffusion, + detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, batch_count, batch_size, cfg_scale, image_cfg_scale, diffusers_guidance_rescale, pag_scale, pag_adaptive, cfg_end, @@ -261,9 +262,14 @@ def select_img2img_tab(tab): (clip_skip, "Clip skip"), (diffusers_guidance_rescale, "CFG rescale"), (full_quality, "Full quality"), - (detailer, "Detailer"), (tiling, "Tiling"), (hidiffusion, "HiDiffusion"), + # detailer + (detailer_enabled, "Detailer"), + (detailer_prompt, "Detailer prompt"), + (detailer_negative, "Detailer negative"), + (detailer_steps, "Detailer steps"), + (detailer_strength, "Detailer strength"), # second pass (enable_hr, "Second pass"), (enable_hr, "Refine"), @@ -282,8 +288,8 @@ def select_img2img_tab(tab): # refiner (refiner_start, "Refiner start"), (refiner_steps, "Refiner steps"), - (refiner_prompt, "Prompt2"), - (refiner_negative, "Negative2"), + (refiner_prompt, "refiner prompt"), + (refiner_negative, "Refiner negative"), # pag (pag_scale, "PAG scale"), (pag_adaptive, "PAG adaptive"), diff --git a/modules/ui_sections.py b/modules/ui_sections.py index 839fa9025..346c53783 100644 --- a/modules/ui_sections.py +++ b/modules/ui_sections.py @@ -63,21 +63,6 @@ def parse_style(styles): def ar_change(ar, width, height): - """ - if ar == 'AR': - return gr.update(interactive=True), gr.update(interactive=True) - try: - (w, h) = [float(x) for x in ar.split(':')] - except Exception as e: - shared.log.warning(f"Invalid aspect ratio: {ar} {e}") - return gr.update(interactive=True), gr.update(interactive=True) - if w > h: - return gr.update(interactive=True, value=width), gr.update(interactive=False, value=int(width * h / w)) - elif w < h: - return gr.update(interactive=False, value=int(height * w / h)), gr.update(interactive=True, value=height) - else: - return gr.update(interactive=True, value=width), gr.update(interactive=False, value=width) - """ if ar == 'AR': return gr.update(), gr.update() try: diff --git a/modules/ui_txt2img.py b/modules/ui_txt2img.py index 9fc69a4db..64d2906f8 100644 --- a/modules/ui_txt2img.py +++ b/modules/ui_txt2img.py @@ -47,7 +47,7 @@ def create_ui(): full_quality, tiling, hidiffusion, _cfg_scale, clip_skip, image_cfg_scale, diffusers_guidance_rescale, pag_scale, pag_adaptive, _cfg_end = ui_sections.create_advanced_inputs('txt2img', base=False) hdr_mode, hdr_brightness, hdr_color, hdr_sharpen, hdr_clamp, hdr_boundary, hdr_threshold, hdr_maximize, hdr_max_center, hdr_max_boundry, hdr_color_picker, hdr_tint_ratio = ui_sections.create_correction_inputs('txt2img') enable_hr, hr_sampler_index, denoising_strength, hr_resize_mode, hr_resize_context, hr_upscaler, hr_force, hr_second_pass_steps, hr_scale, hr_resize_x, hr_resize_y, refiner_steps, refiner_start, refiner_prompt, refiner_negative = ui_sections.create_hires_inputs('txt2img') - detailer = shared.yolo.ui('txt2img') + detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength = shared.yolo.ui('txt2img') override_settings = ui_common.create_override_inputs('txt2img') state = gr.Textbox(value='', visible=False) @@ -64,7 +64,8 @@ def create_ui(): dummy_component, state, txt2img_prompt, txt2img_negative_prompt, txt2img_prompt_styles, steps, sampler_index, hr_sampler_index, - full_quality, detailer, tiling, hidiffusion, + full_quality, tiling, hidiffusion, + detailer_enabled, detailer_prompt, detailer_negative, detailer_steps, detailer_strength, batch_count, batch_size, cfg_scale, image_cfg_scale, diffusers_guidance_rescale, pag_scale, pag_adaptive, cfg_end, clip_skip, @@ -122,9 +123,14 @@ def create_ui(): (image_cfg_scale, "Hires CFG scale"), (diffusers_guidance_rescale, "CFG rescale"), (full_quality, "Full quality"), - (detailer, "Detailer"), (tiling, "Tiling"), (hidiffusion, "HiDiffusion"), + # detailer + (detailer_enabled, "Detailer"), + (detailer_prompt, "Detailer prompt"), + (detailer_negative, "Detailer negative"), + (detailer_steps, "Detailer steps"), + (detailer_strength, "Detailer strength"), # second pass (enable_hr, "Second pass"), (enable_hr, "Refine"), @@ -143,8 +149,8 @@ def create_ui(): # refiner (refiner_start, "Refiner start"), (refiner_steps, "Refiner steps"), - (refiner_prompt, "Prompt2"), - (refiner_negative, "Negative2"), + (refiner_prompt, "refiner prompt"), + (refiner_negative, "Refiner negative"), # pag (pag_scale, "PAG scale"), (pag_adaptive, "PAG adaptive"), diff --git a/scripts/xyz_grid_classes.py b/scripts/xyz_grid_classes.py index cc70d68f8..2b1fa2d59 100644 --- a/scripts/xyz_grid_classes.py +++ b/scripts/xyz_grid_classes.py @@ -1,4 +1,4 @@ -from scripts.xyz_grid_shared import apply_field, apply_task_args, apply_setting, apply_prompt, apply_order, apply_sampler, apply_hr_sampler_name, confirm_samplers, apply_checkpoint, apply_refiner, apply_unet, apply_dict, apply_clip_skip, apply_vae, list_lora, apply_lora, apply_lora_strength, apply_te, apply_styles, apply_upscaler, apply_context, apply_detailer, apply_override, apply_processing, apply_options, apply_seed, format_value_add_label, format_value, format_value_join_list, do_nothing, format_nothing, str_permutations # pylint: disable=no-name-in-module, unused-import +from scripts.xyz_grid_shared import apply_field, apply_task_args, apply_setting, apply_prompt_primary, apply_prompt_refine, apply_prompt_detailer, apply_prompt_all, apply_order, apply_sampler, apply_hr_sampler_name, confirm_samplers, apply_checkpoint, apply_refiner, apply_unet, apply_dict, apply_clip_skip, apply_vae, list_lora, apply_lora, apply_lora_strength, apply_te, apply_styles, apply_upscaler, apply_context, apply_detailer, apply_override, apply_processing, apply_options, apply_seed, format_value_add_label, format_value, format_value_join_list, do_nothing, format_nothing, str_permutations # pylint: disable=no-name-in-module, unused-import from modules import shared, shared_items, sd_samplers, ipadapter, sd_models, sd_vae, sd_unet @@ -93,7 +93,10 @@ def __exit__(self, exc_type, exc_value, tb): AxisOption("[Model] Refiner", str, apply_refiner, cost=0.8, fmt=format_value_add_label, choices=lambda: ['None'] + sorted(sd_models.checkpoints_list)), AxisOption("[Model] Text encoder", str, apply_te, cost=0.7, choices=shared_items.sd_te_items), AxisOption("[Model] Dictionary", str, apply_dict, fmt=format_value_add_label, cost=0.9, choices=lambda: ['None'] + list(sd_models.checkpoints_list)), - AxisOption("[Prompt] Search & replace", str, apply_prompt, fmt=format_value_add_label), + AxisOption("[Prompt] Search & replace", str, apply_prompt_primary, fmt=format_value_add_label), + AxisOption("[Prompt] Search & replace refine", str, apply_prompt_refine, fmt=format_value_add_label), + AxisOption("[Prompt] Search & replace detailer", str, apply_prompt_detailer, fmt=format_value_add_label), + AxisOption("[Prompt] Search & replace all", str, apply_prompt_all, fmt=format_value_add_label), AxisOption("[Prompt] Prompt order", str_permutations, apply_order, fmt=format_value_join_list), AxisOption("[Prompt] Prompt parser", str, apply_setting("prompt_attention"), choices=lambda: ["native", "compel", "xhinker", "a1111", "fixed"]), AxisOption("[Network] LoRA", str, apply_lora, cost=0.5, choices=list_lora), diff --git a/scripts/xyz_grid_on.py b/scripts/xyz_grid_on.py index 5c5ef8bf5..5e03e1c5c 100644 --- a/scripts/xyz_grid_on.py +++ b/scripts/xyz_grid_on.py @@ -286,8 +286,8 @@ def fix_axis_seeds(axis_opt, axis_list): total_steps += p.hr_second_pass_steps * total_jobs else: total_steps *= 2 - if p.detailer: - total_steps += shared.opts.detailer_steps * total_jobs + if p.detailer_enabled: + total_steps += p.detailer_steps * total_jobs total_steps *= p.n_iter total_jobs *= p.n_iter @@ -432,7 +432,7 @@ def cell(x, y, z, ix, iy, iz): def process_images(self, p, *args): # pylint: disable=W0221, W0613 if xyz_results_cache is not None and len(xyz_results_cache.images) > 0: p.restore_faces = False - p.detailer = False + p.detailer_enabled = False p.color_corrections = None # p.scripts = None return xyz_results_cache diff --git a/scripts/xyz_grid_shared.py b/scripts/xyz_grid_shared.py index f9bf26c67..efd70724b 100644 --- a/scripts/xyz_grid_shared.py +++ b/scripts/xyz_grid_shared.py @@ -62,18 +62,37 @@ def apply_seed(p, x, xs): shared.log.debug(f'XYZ grid apply seed: {x}') -def apply_prompt(p, x, xs): +def apply_prompt(positive, negative, p, x, xs): for s in xs: - if s in p.prompt: - shared.log.debug(f'XYZ grid apply prompt: "{s}"="{x}"') - p.prompt = p.prompt.replace(s, x) - if s in p.negative_prompt: - shared.log.debug(f'XYZ grid apply negative: "{s}"="{x}"') - p.negative_prompt = p.negative_prompt.replace(s, x) + shared.log.debug(f'XYZ grid apply prompt: fields={positive}/{negative} "{s}"="{x}"') + orig_positive = getattr(p, positive) + orig_negative = getattr(p, negative) + if s in orig_positive: + setattr(p, positive, orig_positive.replace(s, x)) + if s in orig_negative: + setattr(p, negative, orig_negative.replace(s, x)) + + +def apply_prompt_primary(p, x, xs): + apply_prompt('prompt', 'negative_prompt', p, x, xs) p.all_prompts = None p.all_negative_prompts = None +def apply_prompt_refine(p, x, xs): + apply_prompt('refiner_prompt', 'refiner_negative', p, x, xs) + + +def apply_prompt_detailer(p, x, xs): + apply_prompt('detailer_prompt', 'detailer_negative', p, x, xs) + + +def apply_prompt_all(p, x, xs): + apply_prompt('prompt', 'negative_prompt', p, x, xs) + apply_prompt('refiner_prompt', 'refiner_negative', p, x, xs) + apply_prompt('detailer_prompt', 'detailer_negative', p, x, xs) + + def apply_order(p, x, xs): token_order = [] for token in x: @@ -251,7 +270,7 @@ def apply_detailer(p, opt, x): p.detailer_model = 'GFPGAN' else: is_active = opt in ('true', 'yes', 'y', '1') - p.detailer = is_active + p.detailer_enabled = is_active shared.log.debug(f'XYZ grid apply face-restore: "{x}"') diff --git a/wiki b/wiki index a0c9483d8..7793288e3 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit a0c9483d8a1a979911b056f86afa598fca635857 +Subproject commit 7793288e3ac320f48c728cbee67cd34683c0fa81 From 182000eb74988ae87514bef5a353bf0d90f0eae2 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 7 Jan 2025 10:32:32 -0500 Subject: [PATCH 227/249] settings debug Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 2 +- extensions-builtin/sd-webui-agent-scheduler | 2 +- modules/shared.py | 13 +++++++++---- modules/ui_loadsave.py | 4 ++-- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c5d707b33..d5e9a3c7f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Change Log for SD.Next -## Update for 2025-01-06 +## Update for 2025-01-07 - [Allegro Video](https://huggingface.co/rhymes-ai/Allegro) - optimizations: full offload and quantization support diff --git a/extensions-builtin/sd-webui-agent-scheduler b/extensions-builtin/sd-webui-agent-scheduler index 721a36f59..198e1b498 160000 --- a/extensions-builtin/sd-webui-agent-scheduler +++ b/extensions-builtin/sd-webui-agent-scheduler @@ -1 +1 @@ -Subproject commit 721a36f59507e625c9982397c22edd7c14a0f62a +Subproject commit 198e1b4980ab595468198f1d012f357ff4a31186 diff --git a/modules/shared.py b/modules/shared.py index 238878094..0080433ce 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -1009,6 +1009,7 @@ class Options: data_labels = options_templates filename = None typemap = {int: float} + debug = os.environ.get('SD_CONFIG_DEBUG', None) is not None def __init__(self): self.data = {k: v.default for k, v in self.data_labels.items()} @@ -1022,6 +1023,8 @@ def __setattr__(self, key, value): # pylint: disable=inconsistent-return-stateme if cmd_opts.hide_ui_dir_config and key in restricted_opts: log.warning(f'Settings key is restricted: {key}') return + if self.debug: + log.trace(f'Settings set: {key}={value}') self.data[key] = value return return super(Options, self).__setattr__(key, value) # pylint: disable=super-with-arguments @@ -1079,13 +1082,13 @@ def save_atomic(self, filename=None, silent=False): diff = {} unused_settings = [] - if os.environ.get('SD_CONFIG_DEBUG', None) is not None: + if self.debug: log.debug('Settings: user') for k, v in self.data.items(): log.trace(f' Config: item={k} value={v} default={self.data_labels[k].default if k in self.data_labels else None}') - log.debug('Settings: defaults') - for k in self.data_labels.keys(): - log.trace(f' Setting: item={k} default={self.data_labels[k].default}') + # log.debug('Settings: defaults') + # for k in self.data_labels.keys(): + # log.trace(f' Setting: item={k} default={self.data_labels[k].default}') for k, v in self.data.items(): if k in self.data_labels: @@ -1099,6 +1102,8 @@ def save_atomic(self, filename=None, silent=False): if not k.startswith('uiux_'): unused_settings.append(k) writefile(diff, filename, silent=silent) + if self.debug: + log.trace(f'Settings save: {diff}') if len(unused_settings) > 0: log.debug(f"Settings: unused={unused_settings}") except Exception as err: diff --git a/modules/ui_loadsave.py b/modules/ui_loadsave.py index 6e398603a..01cf2e97c 100644 --- a/modules/ui_loadsave.py +++ b/modules/ui_loadsave.py @@ -4,7 +4,7 @@ from modules.ui_components import ToolButton -debug = os.environ.get('SD_UI_DEBUG', None) +debug_ui = os.environ.get('SD_UI_DEBUG', None) class UiLoadsave: @@ -46,7 +46,7 @@ def apply_field(obj, field, condition=None, init_field=None): setattr(obj, field, saved_value) if init_field is not None: init_field(saved_value) - if debug and key in self.component_mapping and not key.startswith('customscript'): + if debug_ui and key in self.component_mapping and not key.startswith('customscript'): errors.log.warning(f'UI duplicate: key="{key}" id={getattr(obj, "elem_id", None)} class={getattr(obj, "elem_classes", None)}') if field == 'value' and key not in self.component_mapping: self.component_mapping[key] = x From 92fed9f7aad3c68e1f0c3103986f1ccadeb5687a Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 7 Jan 2025 11:04:34 -0500 Subject: [PATCH 228/249] apply settings skip hidden Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 1 + modules/shared.py | 4 ---- modules/ui.py | 2 ++ 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d5e9a3c7f..0e5d3657c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,7 @@ - scheduler api - controlnet with hires - controlnet with batch count + - apply settings skip hidden settings ## Update for 2024-12-31 diff --git a/modules/shared.py b/modules/shared.py index 0080433ce..b1d041df5 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -1078,7 +1078,6 @@ def save_atomic(self, filename=None, silent=False): log.warning(f'Setting: fn="{filename}" save disabled') return try: - # output = json.dumps(self.data, indent=2) diff = {} unused_settings = [] @@ -1086,9 +1085,6 @@ def save_atomic(self, filename=None, silent=False): log.debug('Settings: user') for k, v in self.data.items(): log.trace(f' Config: item={k} value={v} default={self.data_labels[k].default if k in self.data_labels else None}') - # log.debug('Settings: defaults') - # for k in self.data_labels.keys(): - # log.trace(f' Setting: item={k} default={self.data_labels[k].default}') for k, v in self.data.items(): if k in self.data_labels: diff --git a/modules/ui.py b/modules/ui.py index 7f72c716e..8e9a9db85 100644 --- a/modules/ui.py +++ b/modules/ui.py @@ -235,6 +235,8 @@ def run_settings(*args): for key, value, comp in zip(opts.data_labels.keys(), args, components): if comp == dummy_component or value=='dummy': continue + if getattr(comp, 'visible', True) is False: + continue if not opts.same_type(value, opts.data_labels[key].default): log.error(f'Setting bad value: {key}={value} expecting={type(opts.data_labels[key].default).__name__}') continue From 2a86b3bc532aeb358d9112f4d6f9c26b4d835e45 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 7 Jan 2025 12:49:09 -0500 Subject: [PATCH 229/249] hunyuanvideo lora support Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 4 ++++ installer.py | 2 +- modules/lora/extra_networks_lora.py | 26 ++++++++++++++++---------- modules/lora/network_overrides.py | 8 ++++++++ modules/lora/networks.py | 5 +++-- modules/modeldata.py | 2 ++ wiki | 2 +- 7 files changed, 35 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e5d3657c..59993200f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ - [PixelSmith](https://github.com/Thanos-DB/Pixelsmith/) - available for SD-XL in txt2img and img2img workflows - select from *scripts -> pixelsmith* +- [Hunyuan Video](https://github.com/Tencent/HunyuanVideo) LoRA support + - example: - **Logging**: - reverted enable debug by default - updated [debug wiki](https://github.com/vladmandic/automatic/wiki/debug) @@ -47,6 +49,8 @@ - controlnet with hires - controlnet with batch count - apply settings skip hidden settings + - lora diffusers method apply only once + - lora diffusers method set prompt tags and metadata ## Update for 2024-12-31 diff --git a/installer.py b/installer.py index 8d6365462..f10258e8e 100644 --- a/installer.py +++ b/installer.py @@ -459,7 +459,7 @@ def check_python(supported_minors=[9, 10, 11, 12], reason=None): def check_diffusers(): if args.skip_all or args.skip_git: return - sha = 'b5726358cf125f2fa1a596dce321e91a225a57e4' # diffusers commit hash + sha = '03bcf5aefef13a064c34b605e489c0730052cca8' # diffusers commit hash pkg = pkg_resources.working_set.by_key.get('diffusers', None) minor = int(pkg.version.split('.')[1] if pkg is not None else 0) cur = opts.get('diffusers_version', '') if minor > 0 else '' diff --git a/modules/lora/extra_networks_lora.py b/modules/lora/extra_networks_lora.py index e89e57b6e..448b214ac 100644 --- a/modules/lora/extra_networks_lora.py +++ b/modules/lora/extra_networks_lora.py @@ -2,7 +2,7 @@ import os import re import numpy as np -from modules.lora import networks +from modules.lora import networks, network_overrides from modules import extra_networks, shared @@ -155,17 +155,23 @@ def activate(self, p, params_list, step=0, include=[], exclude=[]): fn = f'{sys._getframe(2).f_code.co_name}:{sys._getframe(1).f_code.co_name}' # pylint: disable=protected-access debug_log(f'Load network: type=LoRA include={include} exclude={exclude} requested={requested} fn={fn}') - networks.network_load(names, te_multipliers, unet_multipliers, dyn_dims) # load - has_changed = self.changed(requested, include, exclude) - if has_changed: - networks.network_deactivate(include, exclude) - networks.network_activate(include, exclude) - debug_log(f'Load network: type=LoRA previous={[n.name for n in networks.previously_loaded_networks]} current={[n.name for n in networks.loaded_networks]} changed') - - if len(networks.loaded_networks) > 0 and len(networks.applied_layers) > 0 and step == 0: + force_diffusers = network_overrides.check_override() + if force_diffusers: + has_changed = False # diffusers handle their own loading + if len(exclude) == 0: + networks.network_load(names, te_multipliers, unet_multipliers, dyn_dims) # load only on first call + else: + networks.network_load(names, te_multipliers, unet_multipliers, dyn_dims) # load + has_changed = self.changed(requested, include, exclude) + if has_changed: + networks.network_deactivate(include, exclude) + networks.network_activate(include, exclude) + debug_log(f'Load network: type=LoRA previous={[n.name for n in networks.previously_loaded_networks]} current={[n.name for n in networks.loaded_networks]} changed') + + if len(networks.loaded_networks) > 0 and (len(networks.applied_layers) > 0 or force_diffusers) and step == 0: infotext(p) prompt(p) - if has_changed and len(include) == 0: # print only once + if (has_changed or force_diffusers) and len(include) == 0: # print only once shared.log.info(f'Load network: type=LoRA apply={[n.name for n in networks.loaded_networks]} mode={"fuse" if shared.opts.lora_fuse_diffusers else "backup"} te={te_multipliers} unet={unet_multipliers} time={networks.timer.summary}') def deactivate(self, p): diff --git a/modules/lora/network_overrides.py b/modules/lora/network_overrides.py index 5334f3c1b..81bd07ce1 100644 --- a/modules/lora/network_overrides.py +++ b/modules/lora/network_overrides.py @@ -29,12 +29,17 @@ # 'sd3', 'kandinsky', 'hunyuandit', + 'hunyuanvideo', 'auraflow', ] force_classes = [ # forced always ] +fuse_ignore = [ + 'hunyuanvideo', +] + def check_override(shorthash=''): force = False @@ -47,3 +52,6 @@ def check_override(shorthash=''): if force and shared.opts.lora_maybe_diffusers: shared.log.debug('LoRA override: force diffusers') return force + +def check_fuse(): + return shared.sd_model_type in fuse_ignore diff --git a/modules/lora/networks.py b/modules/lora/networks.py index ebc30b6dd..9e981a234 100644 --- a/modules/lora/networks.py +++ b/modules/lora/networks.py @@ -267,7 +267,8 @@ def network_load(names, te_multipliers=None, unet_multipliers=None, dyn_dims=Non failed_to_load_networks.append(name) shared.log.error(f'Load network: type=LoRA name="{name}" detected={network_on_disk.sd_version if network_on_disk is not None else None} failed') continue - shared.sd_model.embedding_db.load_diffusers_embedding(None, net.bundle_embeddings) + if hasattr(shared.sd_model, 'embedding_db'): + shared.sd_model.embedding_db.load_diffusers_embedding(None, net.bundle_embeddings) net.te_multiplier = te_multipliers[i] if te_multipliers else shared.opts.extra_networks_default_multiplier net.unet_multiplier = unet_multipliers[i] if unet_multipliers else shared.opts.extra_networks_default_multiplier net.dyn_dim = dyn_dims[i] if dyn_dims else shared.opts.extra_networks_default_multiplier @@ -282,7 +283,7 @@ def network_load(names, te_multipliers=None, unet_multipliers=None, dyn_dims=Non try: t0 = time.time() shared.sd_model.set_adapters(adapter_names=diffuser_loaded, adapter_weights=diffuser_scales) - if shared.opts.lora_fuse_diffusers: + if shared.opts.lora_fuse_diffusers and not network_overrides.check_fuse(): shared.sd_model.fuse_lora(adapter_names=diffuser_loaded, lora_scale=1.0, fuse_unet=True, fuse_text_encoder=True) # fuse uses fixed scale since later apply does the scaling shared.sd_model.unload_lora_weights() timer.activate += time.time() - t0 diff --git a/modules/modeldata.py b/modules/modeldata.py index 4b7ec1776..deb4ac49a 100644 --- a/modules/modeldata.py +++ b/modules/modeldata.py @@ -37,6 +37,8 @@ def get_model_type(pipe): model_type = 'cogvideox' elif "Sana" in name: model_type = 'sana' + elif 'HunyuanVideoPipeline' in name: + model_type = 'hunyuanvideo' else: model_type = name return model_type diff --git a/wiki b/wiki index 7793288e3..06094cad4 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 7793288e3ac320f48c728cbee67cd34683c0fa81 +Subproject commit 06094cad465ab26046c2bee0f771ce479c6930c9 From 7a36812a446ff04d37648cfb5acc47f5f4e86173 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 7 Jan 2025 12:58:16 -0500 Subject: [PATCH 230/249] add detailer to modernui Signed-off-by: Vladimir Mandic --- extensions-builtin/sdnext-modernui | 2 +- modules/postprocess/yolo.py | 4 ++-- wiki | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/extensions-builtin/sdnext-modernui b/extensions-builtin/sdnext-modernui index 62d8a54a7..74e7476bd 160000 --- a/extensions-builtin/sdnext-modernui +++ b/extensions-builtin/sdnext-modernui @@ -1 +1 @@ -Subproject commit 62d8a54a7ec24a6fbf69697da18e67035754d072 +Subproject commit 74e7476bda2dae175fcff604d6149913a10a87a3 diff --git a/modules/postprocess/yolo.py b/modules/postprocess/yolo.py index 44016c18b..57020b60f 100644 --- a/modules/postprocess/yolo.py +++ b/modules/postprocess/yolo.py @@ -328,9 +328,9 @@ def ui_settings_change(detailers, classes, strength, padding, blur, min_confiden with gr.Row(): classes = gr.Textbox(label="Classes", placeholder="Classes", elem_id=f"{tab}_detailer_classes") with gr.Row(): - prompt = gr.Textbox(label="Detailer prompt", value='', placeholder='Detailer prompt', lines=2) + prompt = gr.Textbox(label="Detailer prompt", value='', placeholder='Detailer prompt', lines=2, elem_id=f"{tab}_detailer_prompt") with gr.Row(): - negative = gr.Textbox(label="Detailer negative prompt", value='', placeholder='Detailer negative prompt', lines=2) + negative = gr.Textbox(label="Detailer negative prompt", value='', placeholder='Detailer negative prompt', lines=2, elem_id=f"{tab}_detailer_negative") with gr.Row(): steps = gr.Slider(label="Detailer steps", elem_id=f"{tab}_detailer_steps", value=10, min=0, max=99, step=1) strength = gr.Slider(label="Detailer strength", elem_id=f"{tab}_detailer_strength", value=0.3, minimum=0, maximum=1, step=0.01) diff --git a/wiki b/wiki index 06094cad4..ba50d70e4 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 06094cad465ab26046c2bee0f771ce479c6930c9 +Subproject commit ba50d70e474d0ff06fe3ac24c18c192bdf210359 From ff5cb92b52fb67354854225151fef051ccb98a64 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Tue, 7 Jan 2025 17:13:14 -0500 Subject: [PATCH 231/249] flux bnb on-the-fly for unet-only Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 1 + modules/k-diffusion | 2 +- modules/model_flux.py | 34 ++++++++++++++++++++++------------ modules/model_flux_nf4.py | 18 ++++-------------- modules/sd_models.py | 2 +- 5 files changed, 29 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 59993200f..4edcf4603 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,6 +51,7 @@ - apply settings skip hidden settings - lora diffusers method apply only once - lora diffusers method set prompt tags and metadata + - flux support on-the-fly quantization for bnb of unet only ## Update for 2024-12-31 diff --git a/modules/k-diffusion b/modules/k-diffusion index 21d12c91a..8018de0b4 160000 --- a/modules/k-diffusion +++ b/modules/k-diffusion @@ -1 +1 @@ -Subproject commit 21d12c91ad4550e8fcf3308ff9fe7116b3f19a08 +Subproject commit 8018de0b43da8d66617f3ef10d3f2a41c1d78836 diff --git a/modules/model_flux.py b/modules/model_flux.py index 9b42de3e7..f8d112953 100644 --- a/modules/model_flux.py +++ b/modules/model_flux.py @@ -84,7 +84,7 @@ def load_flux_bnb(checkpoint_info, diffusers_load_config): # pylint: disable=unu repo_path = checkpoint_info else: repo_path = checkpoint_info.path - model_quant.load_bnb('Load model: type=T5') + model_quant.load_bnb('Load model: type=FLUX') quant = model_quant.get_quant(repo_path) try: if quant == 'fp8': @@ -203,7 +203,8 @@ def load_transformer(file_path): # triggered by opts.sd_unet change "torch_dtype": devices.dtype, "cache_dir": shared.opts.hfcache_dir, } - shared.log.info(f'Load module: type=UNet/Transformer file="{file_path}" offload={shared.opts.diffusers_offload_mode} quant={quant} dtype={devices.dtype}') + if quant is not None and quant != 'none': + shared.log.info(f'Load module: type=UNet/Transformer file="{file_path}" offload={shared.opts.diffusers_offload_mode} prequant={quant} dtype={devices.dtype}') if 'gguf' in file_path.lower(): # _transformer, _text_encoder_2 = load_flux_gguf(file_path) from modules import ggml @@ -214,25 +215,34 @@ def load_transformer(file_path): # triggered by opts.sd_unet change _transformer, _text_encoder_2 = load_flux_quanto(file_path) if _transformer is not None: transformer = _transformer - elif quant == 'fp8' or quant == 'fp4' or quant == 'nf4' or 'Model' in shared.opts.bnb_quantization: + elif quant == 'fp8' or quant == 'fp4' or quant == 'nf4': _transformer, _text_encoder_2 = load_flux_bnb(file_path, diffusers_load_config) if _transformer is not None: transformer = _transformer elif 'nf4' in quant: # TODO flux: fix loader for civitai nf4 models from modules.model_flux_nf4 import load_flux_nf4 - _transformer, _text_encoder_2 = load_flux_nf4(file_path) + _transformer, _text_encoder_2 = load_flux_nf4(file_path, prequantized=True) if _transformer is not None: transformer = _transformer else: - quant_args = {} - quant_args = model_quant.create_bnb_config(quant_args) + quant_args = model_quant.create_bnb_config({}) if quant_args: - model_quant.load_bnb(f'Load model: type=Sana quant={quant_args}') - if not quant_args: - quant_args = model_quant.create_ao_config(quant_args) - if quant_args: - model_quant.load_torchao(f'Load model: type=Sana quant={quant_args}') - transformer = diffusers.FluxTransformer2DModel.from_single_file(file_path, **diffusers_load_config, **quant_args) + model_quant.load_bnb(f'Load model: type=FLUX quant={quant_args}') + shared.log.info(f'Load module: type=UNet/Transformer file="{file_path}" offload={shared.opts.diffusers_offload_mode} quant=bnb dtype={devices.dtype}') + from modules.model_flux_nf4 import load_flux_nf4 + transformer, _text_encoder_2 = load_flux_nf4(file_path, prequantized=False) + if transformer is not None: + return transformer + quant_args = model_quant.create_ao_config({}) + if quant_args: + model_quant.load_torchao(f'Load model: type=FLUX quant={quant_args}') + shared.log.info(f'Load module: type=UNet/Transformer file="{file_path}" offload={shared.opts.diffusers_offload_mode} quant=torchao dtype={devices.dtype}') + transformer = diffusers.FluxTransformer2DModel.from_single_file(file_path, **diffusers_load_config, **quant_args) + if transformer is not None: + return transformer + shared.log.info(f'Load module: type=UNet/Transformer file="{file_path}" offload={shared.opts.diffusers_offload_mode} quant=none dtype={devices.dtype}') + # shared.log.warning('Load module: type=UNet/Transformer does not support load-time quantization') # TODO flux transformer from-single-file with quant + transformer = diffusers.FluxTransformer2DModel.from_single_file(file_path, **diffusers_load_config) if transformer is None: shared.log.error('Failed to load UNet model') shared.opts.sd_unet = 'None' diff --git a/modules/model_flux_nf4.py b/modules/model_flux_nf4.py index d023907d6..b00c3320e 100644 --- a/modules/model_flux_nf4.py +++ b/modules/model_flux_nf4.py @@ -24,7 +24,6 @@ def _replace_with_bnb_linear( ): """ Private method that wraps the recursion for module replacement. - Returns the converted model and a boolean that indicates if the conversion has been successfull or not. """ bnb = model_quant.load_bnb('Load model: type=FLUX') @@ -106,7 +105,6 @@ def create_quantized_param( new_value = old_value.to(target_device) else: new_value = param_value.to(target_device) - new_value = torch.nn.Parameter(new_value, requires_grad=old_value.requires_grad) module._parameters[tensor_name] = new_value # pylint: disable=protected-access return @@ -121,13 +119,8 @@ def create_quantized_param( raise ValueError(f"{tensor_name} is on the meta device, we need a `value` to put in on {target_device}.") if pre_quantized: - if (param_name + ".quant_state.bitsandbytes__fp4" not in state_dict) and ( - param_name + ".quant_state.bitsandbytes__nf4" not in state_dict - ): - raise ValueError( - f"Supplied state dict for {param_name} does not contain `bitsandbytes__*` and possibly other `quantized_stats` components." - ) - + if (param_name + ".quant_state.bitsandbytes__fp4" not in state_dict) and (param_name + ".quant_state.bitsandbytes__nf4" not in state_dict): + raise ValueError(f"Supplied state dict for {param_name} does not contain `bitsandbytes__*` and possibly other `quantized_stats` components.") quantized_stats = {} for k, v in state_dict.items(): # `startswith` to counter for edge cases where `param_name` @@ -136,23 +129,20 @@ def create_quantized_param( quantized_stats[k] = v if unexpected_keys is not None and k in unexpected_keys: unexpected_keys.remove(k) - new_value = bnb.nn.Params4bit.from_prequantized( data=param_value, quantized_stats=quantized_stats, requires_grad=False, device=target_device, ) - else: new_value = param_value.to("cpu") kwargs = old_value.__dict__ new_value = bnb.nn.Params4bit(new_value, requires_grad=False, **kwargs).to(target_device) - module._parameters[tensor_name] = new_value # pylint: disable=protected-access -def load_flux_nf4(checkpoint_info): +def load_flux_nf4(checkpoint_info, prequantized: bool = True): transformer = None text_encoder_2 = None if isinstance(checkpoint_info, str): @@ -197,7 +187,7 @@ def load_flux_nf4(checkpoint_info): if not check_quantized_param(transformer, param_name): set_module_tensor_to_device(transformer, param_name, device=0, value=param) else: - create_quantized_param(transformer, param, param_name, target_device=0, state_dict=original_state_dict, pre_quantized=True) + create_quantized_param(transformer, param, param_name, target_device=0, state_dict=original_state_dict, pre_quantized=prequantized) except Exception as e: transformer, text_encoder_2 = None, None shared.log.error(f"Load model: type=FLUX failed to load UNET: {e}") diff --git a/modules/sd_models.py b/modules/sd_models.py index 8b0eb0a89..1916fa7ec 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -1613,7 +1613,7 @@ def disable_offload(sd_model): def clear_caches(): - shared.log.debug('Cache clear') + # shared.log.debug('Cache clear') if not shared.opts.lora_legacy: from modules.lora import networks networks.loaded_networks.clear() From 4bd5c17426c164f08dbeec4f90d95e5900355df4 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 8 Jan 2025 08:22:53 -0500 Subject: [PATCH 232/249] fix pbar error Signed-off-by: Vladimir Mandic --- extensions-builtin/sd-webui-agent-scheduler | 2 +- modules/processing_args.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/extensions-builtin/sd-webui-agent-scheduler b/extensions-builtin/sd-webui-agent-scheduler index 198e1b498..cd878626f 160000 --- a/extensions-builtin/sd-webui-agent-scheduler +++ b/extensions-builtin/sd-webui-agent-scheduler @@ -1 +1 @@ -Subproject commit 198e1b4980ab595468198f1d012f357ff4a31186 +Subproject commit cd878626f3b4f9a0c7c45c7d70b73a6168f612a4 diff --git a/modules/processing_args.py b/modules/processing_args.py index 2bc1c1ce2..e73cda85c 100644 --- a/modules/processing_args.py +++ b/modules/processing_args.py @@ -108,7 +108,10 @@ def set_pipeline_args(p, model, prompts:list, negative_prompts:list, prompts_2:t shared.sd_model = sd_models.apply_balanced_offload(shared.sd_model) apply_circular(p.tiling, model) if hasattr(model, "set_progress_bar_config"): - model.set_progress_bar_config(bar_format='Progress {rate_fmt}{postfix} {bar} {percentage:3.0f}% {n_fmt}/{total_fmt} {elapsed} {remaining} ' + '\x1b[38;5;71m' + desc, ncols=80, colour='#327fba', disable=disable_pbar) + if disable_pbar: + model.set_progress_bar_config(bar_format='Progress {rate_fmt}{postfix} {bar} {percentage:3.0f}% {n_fmt}/{total_fmt} {elapsed} {remaining} ' + '\x1b[38;5;71m' + desc, ncols=80, colour='#327fba', disable=disable_pbar) + else: + model.set_progress_bar_config(bar_format='Progress {rate_fmt}{postfix} {bar} {percentage:3.0f}% {n_fmt}/{total_fmt} {elapsed} {remaining} ' + '\x1b[38;5;71m' + desc, ncols=80, colour='#327fba') args = {} has_vae = hasattr(model, 'vae') or (hasattr(model, 'pipe') and hasattr(model.pipe, 'vae')) if hasattr(model, 'pipe') and not hasattr(model, 'no_recurse'): # recurse From 4ef1eeccd87aff743bdd080fbebd0745218d4835 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 8 Jan 2025 09:35:49 -0500 Subject: [PATCH 233/249] startup tracing Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 3 +- installer.py | 103 +++++++++++++++++++++++++++--------- launch.py | 13 ++++- modules/script_callbacks.py | 2 +- modules/timer.py | 16 ++++-- webui.py | 12 ++++- 6 files changed, 117 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4edcf4603..640b1f300 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Change Log for SD.Next -## Update for 2025-01-07 +## Update for 2025-01-08 - [Allegro Video](https://huggingface.co/rhymes-ai/Allegro) - optimizations: full offload and quantization support @@ -25,6 +25,7 @@ - **Refactor**: - refactored progress monitoring, job updates and live preview - improved metadata save and restore + - startup tracing and optimizations - **Schedulers**: - [TDD](https://github.com/RedAIGC/Target-Driven-Distillation) new super-fast scheduler that can generate images in 4-8 steps recommended to use with [TDD LoRA](https://huggingface.co/RED-AIGC/TDD/tree/main) diff --git a/installer.py b/installer.py index f10258e8e..96259da36 100644 --- a/installer.py +++ b/installer.py @@ -8,6 +8,7 @@ import platform import subprocess import cProfile +import importlib # pylint: disable=deprecated-module class Dot(dict): # dot notation access to dictionary attributes @@ -58,6 +59,14 @@ class Dot(dict): # dot notation access to dictionary attributes # 'stable-diffusion-webui-images-browser': '27fe4a7', } + +try: + from modules.timer import init + ts = init.ts +except Exception: + ts = lambda *args, **kwargs: None # pylint: disable=unnecessary-lambda-assignment + + # setup console and file logging def setup_logging(): @@ -84,6 +93,7 @@ def emit(self, record): def get(self): return self.buffer + t_start = time.time() from functools import partial, partialmethod from logging.handlers import RotatingFileHandler from rich.theme import Theme @@ -148,6 +158,7 @@ def get(self): logging.getLogger("ControlNet").handlers = log.handlers logging.getLogger("lycoris").handlers = log.handlers # logging.getLogger("DeepSpeed").handlers = log.handlers + ts('log', t_start) def get_logfile(): @@ -201,11 +212,11 @@ def package_spec(package): # check if package is installed @lru_cache() def installed(package, friendly: str = None, reload = False, quiet = False): + t_start = time.time() ok = True try: if reload: try: - import importlib # pylint: disable=deprecated-module importlib.reload(pkg_resources) except Exception: pass @@ -237,13 +248,15 @@ def installed(package, friendly: str = None, reload = False, quiet = False): else: if not quiet: log.debug(f'Install: package="{p[0]}" install required') + ts('installed', t_start) return ok except Exception as e: log.error(f'Install: package="{pkgs}" {e}') + ts('installed', t_start) return False - def uninstall(package, quiet = False): + t_start = time.time() packages = package if isinstance(package, list) else [package] res = '' for p in packages: @@ -251,11 +264,13 @@ def uninstall(package, quiet = False): if not quiet: log.warning(f'Package: {p} uninstall') res += pip(f"uninstall {p} --yes --quiet", ignore=True, quiet=True, uv=False) + ts('uninstall', t_start) return res @lru_cache() def pip(arg: str, ignore: bool = False, quiet: bool = True, uv = True): + t_start = time.time() originalArg = arg arg = arg.replace('>=', '==') package = arg.replace("install", "").replace("--upgrade", "").replace("--no-deps", "").replace("--force", "").replace(" ", " ").strip() @@ -283,12 +298,14 @@ def pip(arg: str, ignore: bool = False, quiet: bool = True, uv = True): errors.append(f'pip: {package}') log.error(f'Install: {pipCmd}: {arg}') log.debug(f'Install: pip output {txt}') + ts('pip', t_start) return txt # install package using pip if not already installed @lru_cache() def install(package, friendly: str = None, ignore: bool = False, reinstall: bool = False, no_deps: bool = False, quiet: bool = False): + t_start = time.time() res = '' if args.reinstall or args.upgrade: global quick_allowed # pylint: disable=global-statement @@ -297,16 +314,17 @@ def install(package, friendly: str = None, ignore: bool = False, reinstall: bool deps = '' if not no_deps else '--no-deps ' res = pip(f"install{' --upgrade' if not args.uv else ''} {deps}{package}", ignore=ignore, uv=package != "uv" and not package.startswith('git+')) try: - import importlib # pylint: disable=deprecated-module importlib.reload(pkg_resources) except Exception: pass + ts('install', t_start) return res # execute git command @lru_cache() def git(arg: str, folder: str = None, ignore: bool = False, optional: bool = False): + t_start = time.time() if args.skip_git: return '' if optional: @@ -328,6 +346,7 @@ def git(arg: str, folder: str = None, ignore: bool = False, optional: bool = Fal if 'or stash them' in txt: log.error(f'Git local changes detected: check details log="{log_file}"') log.debug(f'Git output: {txt}') + ts('git', t_start) return txt @@ -335,6 +354,7 @@ def git(arg: str, folder: str = None, ignore: bool = False, optional: bool = Fal def branch(folder=None): # if args.experimental: # return None + t_start = time.time() if not os.path.exists(os.path.join(folder or os.curdir, '.git')): return None branches = [] @@ -357,11 +377,13 @@ def branch(folder=None): b = b.split('\n')[0].replace('*', '').strip() log.debug(f'Git submodule: {folder} / {b}') git(f'checkout {b}', folder, ignore=True, optional=True) + ts('branch', t_start) return b # update git repository def update(folder, keep_branch = False, rebase = True): + t_start = time.time() try: git('config rebase.Autostash true') except Exception: @@ -383,11 +405,13 @@ def update(folder, keep_branch = False, rebase = True): if commit is not None: res = git(f'checkout {commit}', folder) debug(f'Install update: folder={folder} branch={b} args={arg} commit={commit} {res}') + ts('update', t_start) return res # clone git repository def clone(url, folder, commithash=None): + t_start = time.time() if os.path.exists(folder): if commithash is None: update(folder) @@ -403,6 +427,7 @@ def clone(url, folder, commithash=None): git(f'clone "{url}" "{folder}"') if commithash is not None: git(f'-C "{folder}" checkout {commithash}') + ts('clone', t_start) def get_platform(): @@ -427,6 +452,7 @@ def get_platform(): # check python version def check_python(supported_minors=[9, 10, 11, 12], reason=None): + t_start = time.time() if args.quick: return log.info(f'Python: version={platform.python_version()} platform={platform.system()} bin="{sys.executable}" venv="{sys.prefix}"') @@ -453,10 +479,12 @@ def check_python(supported_minors=[9, 10, 11, 12], reason=None): else: git_version = git('--version', folder=None, ignore=False) log.debug(f'Git: version={git_version.replace("git version", "").strip()}') + ts('python', t_start) # check diffusers version def check_diffusers(): + t_start = time.time() if args.skip_all or args.skip_git: return sha = '03bcf5aefef13a064c34b605e489c0730052cca8' # diffusers commit hash @@ -472,42 +500,30 @@ def check_diffusers(): pip(f'install --upgrade git+https://github.com/huggingface/diffusers@{sha}', ignore=False, quiet=True, uv=False) global diffusers_commit # pylint: disable=global-statement diffusers_commit = sha + ts('diffusers', t_start) # check onnx version def check_onnx(): + t_start = time.time() if args.skip_all or args.skip_requirements: return if not installed('onnx', quiet=True): install('onnx', 'onnx', ignore=True) if not installed('onnxruntime', quiet=True) and not (installed('onnxruntime-gpu', quiet=True) or installed('onnxruntime-openvino', quiet=True) or installed('onnxruntime-training', quiet=True)): # allow either install('onnxruntime', 'onnxruntime', ignore=True) - - -def check_torchao(): - """ - if args.skip_all or args.skip_requirements: - return - if installed('torchao', quiet=True): - ver = package_version('torchao') - if ver != '0.5.0': - log.debug(f'Uninstall: torchao=={ver}') - pip('uninstall --yes torchao', ignore=True, quiet=True, uv=False) - for m in [m for m in sys.modules if m.startswith('torchao')]: - del sys.modules[m] - """ - return + ts('onnx', t_start) def install_cuda(): + t_start = time.time() log.info('CUDA: nVidia toolkit detected') - if not (args.skip_all or args.skip_requirements): - install('onnxruntime-gpu', 'onnxruntime-gpu', ignore=True, quiet=True) - # return os.environ.get('TORCH_COMMAND', 'torch torchvision --index-url https://download.pytorch.org/whl/cu124') + ts('cuda', t_start) return os.environ.get('TORCH_COMMAND', 'torch==2.5.1+cu124 torchvision==0.20.1+cu124 --index-url https://download.pytorch.org/whl/cu124') def install_rocm_zluda(): + t_start = time.time() if args.skip_all or args.skip_requirements: return None from modules import rocm @@ -628,10 +644,12 @@ def install_rocm_zluda(): else: log.warning(f'ROCm: device={device.name} could not auto-detect HSA version') + ts('amd', t_start) return torch_command def install_ipex(torch_command): + t_start = time.time() check_python(supported_minors=[10,11], reason='IPEX backend requires Python 3.10 or 3.11') args.use_ipex = True # pylint: disable=attribute-defined-outside-init log.info('IPEX: Intel OneAPI toolkit detected') @@ -650,10 +668,12 @@ def install_ipex(torch_command): install(os.environ.get('OPENVINO_PACKAGE', 'openvino==2024.5.0'), 'openvino', ignore=True) install('nncf==2.7.0', ignore=True, no_deps=True) # requires older pandas install(os.environ.get('ONNXRUNTIME_PACKAGE', 'onnxruntime-openvino'), 'onnxruntime-openvino', ignore=True) + ts('ipex', t_start) return torch_command def install_openvino(torch_command): + t_start = time.time() check_python(supported_minors=[9, 10, 11, 12], reason='OpenVINO backend requires Python 3.9, 3.10 or 3.11') log.info('OpenVINO: selected') if sys.platform == 'darwin': @@ -668,10 +688,12 @@ def install_openvino(torch_command): os.environ.setdefault('NEOReadDebugKeys', '1') if os.environ.get("ClDeviceGlobalMemSizeAvailablePercent", None) is None: os.environ.setdefault('ClDeviceGlobalMemSizeAvailablePercent', '100') + ts('openvino', t_start) return torch_command def install_torch_addons(): + t_start = time.time() xformers_package = os.environ.get('XFORMERS_PACKAGE', '--pre xformers') if opts.get('cross_attention_optimization', '') == 'xFormers' or args.use_xformers else 'none' triton_command = os.environ.get('TRITON_COMMAND', 'triton') if sys.platform == 'linux' else None if 'xformers' in xformers_package: @@ -697,10 +719,12 @@ def install_torch_addons(): uninstall('wandb', quiet=True) if triton_command is not None: install(triton_command, 'triton', quiet=True) + ts('addons', t_start) # check torch version def check_torch(): + t_start = time.time() if args.skip_torch: log.info('Torch: skip tests') return @@ -812,10 +836,12 @@ def check_torch(): if args.profile: pr.disable() print_profile(pr, 'Torch') + ts('torch', t_start) # check modified files def check_modified_files(): + t_start = time.time() if args.quick: return if args.skip_git: @@ -832,10 +858,12 @@ def check_modified_files(): log.warning(f'Modified files: {files}') except Exception: pass + ts('files', t_start) # install required packages def install_packages(): + t_start = time.time() if args.profile: pr = cProfile.Profile() pr.enable() @@ -850,6 +878,7 @@ def install_packages(): if args.profile: pr.disable( ) print_profile(pr, 'Packages') + ts('packages', t_start) # run extension installer @@ -873,6 +902,7 @@ def run_extension_installer(folder): except Exception as e: log.error(f'Extension installer exception: {e}') + # get list of all enabled extensions def list_extensions_folder(folder, quiet=False): name = os.path.basename(folder) @@ -888,6 +918,7 @@ def list_extensions_folder(folder, quiet=False): # run installer for each installed and enabled extension and optionally update them def install_extensions(force=False): + t_start = time.time() if args.profile: pr = cProfile.Profile() pr.enable() @@ -936,11 +967,13 @@ def install_extensions(force=False): if args.profile: pr.disable() print_profile(pr, 'Extensions') + ts('extensions', t_start) return '\n'.join(res) # initialize and optionally update submodules def install_submodules(force=True): + t_start = time.time() if args.profile: pr = cProfile.Profile() pr.enable() @@ -968,10 +1001,12 @@ def install_submodules(force=True): if args.profile: pr.disable() print_profile(pr, 'Submodule') + ts('submodules', t_start) return '\n'.join(res) def ensure_base_requirements(): + t_start = time.time() setuptools_version = '69.5.1' def update_setuptools(): @@ -979,7 +1014,6 @@ def update_setuptools(): global pkg_resources, setuptools, distutils # pylint: disable=global-statement # python may ship with incompatible setuptools subprocess.run(f'"{sys.executable}" -m pip install setuptools=={setuptools_version}', shell=True, check=False, env=os.environ, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - import importlib # need to delete all references to modules to be able to reload them otherwise python will use cached version modules = [m for m in sys.modules if m.startswith('setuptools') or m.startswith('pkg_resources') or m.startswith('distutils')] for m in modules: @@ -1004,9 +1038,11 @@ def update_setuptools(): install('rich', 'rich', quiet=True) install('psutil', 'psutil', quiet=True) install('requests', 'requests', quiet=True) + ts('base', t_start) def install_optional(): + t_start = time.time() log.info('Installing optional requirements...') install('basicsr') install('gfpgan') @@ -1027,9 +1063,11 @@ def install_optional(): os.rename(scripts_dir, scripts_dir + '_gguf') except Exception: pass + ts('optional', t_start) def install_requirements(): + t_start = time.time() if args.profile: pr = cProfile.Profile() pr.enable() @@ -1053,6 +1091,7 @@ def install_requirements(): if args.profile: pr.disable() print_profile(pr, 'Requirements') + ts('requirements', t_start) # set environment variables controling the behavior of various libraries @@ -1117,14 +1156,15 @@ def check_extensions(): for f in os.listdir(extension_dir): if '.json' in f or '.csv' in f or '__pycache__' in f: continue - ts = os.path.getmtime(os.path.join(extension_dir, f)) - newest = max(newest, ts) + mtime = os.path.getmtime(os.path.join(extension_dir, f)) + newest = max(newest, mtime) newest_all = max(newest_all, newest) # log.debug(f'Extension version: {time.ctime(newest)} {folder}{os.pathsep}{ext}') return round(newest_all) def get_version(force=False): + t_start = time.time() global version # pylint: disable=global-statement if version is None or force: try: @@ -1159,6 +1199,7 @@ def get_version(force=False): except Exception: os.chdir(cwd) version['ui'] = 'unknown' + ts('version', t_start) return version @@ -1168,6 +1209,7 @@ def same(ver): ui = ver['ui'] if ver is not None and 'ui' in ver else 'unknown' return core == ui or (core == 'master' and ui == 'main') + t_start = time.time() if not same(ver): log.debug(f'Branch mismatch: sdnext={ver["branch"]} ui={ver["ui"]}') cwd = os.getcwd() @@ -1184,6 +1226,7 @@ def same(ver): except Exception as e: log.debug(f'Branch switch: {e}') os.chdir(cwd) + ts('ui', t_start) def check_venv(): @@ -1193,6 +1236,7 @@ def try_relpath(p): except ValueError: return p + t_start = time.time() import site pkg_path = [try_relpath(p) for p in site.getsitepackages() if os.path.exists(p)] log.debug(f'Packages: venv={try_relpath(sys.prefix)} site={pkg_path}') @@ -1212,10 +1256,12 @@ def try_relpath(p): os.unlink(fn) except Exception as e: log.error(f'Packages: site={p} invalid={f} error={e}') + ts('venv', t_start) # check version of the main repo and optionally upgrade it def check_version(offline=False, reset=True): # pylint: disable=unused-argument + t_start = time.time() if args.skip_all: return if not os.path.exists('.git'): @@ -1261,15 +1307,18 @@ def check_version(offline=False, reset=True): # pylint: disable=unused-argument log.info(f'Repository latest available {commits["commit"]["sha"]} {commits["commit"]["commit"]["author"]["date"]}') except Exception as e: log.error(f'Repository failed to check version: {e} {commits}') + ts('latest', t_start) def update_wiki(): + t_start = time.time() if args.upgrade: log.info('Updating Wiki') try: update(os.path.join(os.path.dirname(__file__), "wiki")) except Exception: log.error('Wiki update error') + ts('wiki', t_start) # check if we can run setup in quick mode @@ -1358,6 +1407,7 @@ def parse_args(parser): def extensions_preload(parser): + t_start = time.time() if args.profile: pr = cProfile.Profile() pr.enable() @@ -1379,9 +1429,11 @@ def extensions_preload(parser): if args.profile: pr.disable() print_profile(pr, 'Preload') + ts('preload', t_start) def git_reset(folder='.'): + t_start = time.time() log.warning('Running GIT reset') global quick_allowed # pylint: disable=global-statement quick_allowed = False @@ -1397,9 +1449,11 @@ def git_reset(folder='.'): git('submodule update --init --recursive') git('submodule sync --recursive') log.info('GIT reset complete') + ts('reset', t_start) def read_options(): + t_start = time.time() global opts # pylint: disable=global-statement if os.path.isfile(args.config): with open(args.config, "r", encoding="utf8") as file: @@ -1409,3 +1463,4 @@ def read_options(): opts = json.loads(opts) except Exception as e: log.error(f'Error reading options file: {file} {e}') + ts('options', t_start) diff --git a/launch.py b/launch.py index e00da58c7..07fa93697 100755 --- a/launch.py +++ b/launch.py @@ -24,12 +24,20 @@ skip_install = False # parsed by some extensions +try: + from modules.timer import launch + rec = launch.record +except Exception: + rec = lambda *args, **kwargs: None # pylint: disable=unnecessary-lambda-assignment + + def init_args(): global parser, args # pylint: disable=global-statement import modules.cmd_args parser = modules.cmd_args.parser installer.add_args(parser) args, _ = parser.parse_known_args() + rec('args') def init_paths(): @@ -38,6 +46,7 @@ def init_paths(): modules.paths.register_paths() script_path = modules.paths.script_path extensions_dir = modules.paths.extensions_dir + rec('paths') def get_custom_args(): @@ -60,6 +69,7 @@ def get_custom_args(): ldd = os.environ.get('LD_PRELOAD', None) if ldd is not None: installer.log.debug(f'Linker flags: "{ldd}"') + rec('args') @lru_cache() @@ -71,6 +81,7 @@ def commit_hash(): # compatbility function stored_commit_hash = run(f"{git} rev-parse HEAD").strip() except Exception: stored_commit_hash = "" + rec('commit') return stored_commit_hash @@ -185,6 +196,7 @@ def start_server(immediate=True, server=None): if args.profile: pr.disable() installer.print_profile(pr, 'WebUI') + rec('server') return uvicorn, server @@ -218,7 +230,6 @@ def main(): installer.install("uv", "uv") installer.check_torch() installer.check_onnx() - installer.check_torchao() installer.check_diffusers() installer.check_modified_files() if args.reinstall: diff --git a/modules/script_callbacks.py b/modules/script_callbacks.py index 6c91edf44..a0c85a283 100644 --- a/modules/script_callbacks.py +++ b/modules/script_callbacks.py @@ -141,7 +141,7 @@ def print_timers(): if v > 0.05: long_callbacks.append(f'{k}={v:.2f}') if len(long_callbacks) > 0: - errors.log.debug(f'Script callback init time: {" ".join(long_callbacks)}') + errors.log.debug(f'Script init: {long_callbacks}') def clear_callbacks(): diff --git a/modules/timer.py b/modules/timer.py index 3c7d42f6d..69107e605 100644 --- a/modules/timer.py +++ b/modules/timer.py @@ -25,9 +25,12 @@ def elapsed(self, reset=True): def add(self, name, t): if name not in self.records: - self.records[name] = t - else: - self.records[name] += t + self.records[name] = 0 + self.records[name] += t + + def ts(self, name, t): + elapsed = time.time() - t + self.add(name, elapsed) def record(self, category=None, extra_time=0, reset=True): e = self.elapsed(reset) @@ -41,6 +44,8 @@ def record(self, category=None, extra_time=0, reset=True): def summary(self, min_time=default_min_time, total=True): if self.profile: min_time = -1 + if self.total <= 0: + self.total = sum(self.records.values()) res = f"total={self.total:.2f} " if total else '' additions = [x for x in self.records.items() if x[1] >= min_time] additions = sorted(additions, key=lambda x: x[1], reverse=True) @@ -49,6 +54,9 @@ def summary(self, min_time=default_min_time, total=True): res += " ".join([f"{category}={time_taken:.2f}" for category, time_taken in additions]) return res + def get_total(self): + return sum(self.records.values()) + def dct(self, min_time=default_min_time): if self.profile: res = {k: round(v, 4) for k, v in self.records.items()} @@ -61,3 +69,5 @@ def reset(self): startup = Timer() process = Timer() +launch = Timer() +init = Timer() diff --git a/webui.py b/webui.py index 33769994e..2c70192f1 100644 --- a/webui.py +++ b/webui.py @@ -171,7 +171,7 @@ def sigint_handler(_sig, _frame): def load_model(): if not shared.opts.sd_checkpoint_autoload and shared.cmd_opts.ckpt is None: - log.debug('Model auto load disabled') + log.info('Model auto load disabled') else: shared.state.begin('Load') thread_model = Thread(target=lambda: shared.sd_model) @@ -356,7 +356,15 @@ def webui(restart=False): for m in modules.scripts.postprocessing_scripts_data: debug(f' {m}') modules.script_callbacks.print_timers() - log.info(f"Startup time: {timer.startup.summary()}") + + if cmd_opts.profile: + log.info(f"Launch time: {timer.launch.summary(min_time=0)}") + log.info(f"Installer time: {timer.init.summary(min_time=0)}") + log.info(f"Startup time: {timer.startup.summary(min_time=0)}") + else: + timer.startup.add('launch', timer.launch.get_total()) + timer.startup.add('installer', timer.launch.get_total()) + log.info(f"Startup time: {timer.startup.summary()}") timer.startup.reset() if not restart: From 61d29ab7ed2926a42e05d893659cc7353f218ac4 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Wed, 8 Jan 2025 10:40:49 -0500 Subject: [PATCH 234/249] correct log message Signed-off-by: Vladimir Mandic --- modules/shared.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/shared.py b/modules/shared.py index b1d041df5..3c83f5d67 100644 --- a/modules/shared.py +++ b/modules/shared.py @@ -438,14 +438,14 @@ def get_default_modes(): if gpu_memory <= 4: cmd_opts.lowvram = True default_offload_mode = "sequential" - log.info(f"Device detect: memory={gpu_memory:.1f} optimization=lowvram") + log.info(f"Device detect: memory={gpu_memory:.1f} default=sequential optimization=lowvram") # elif gpu_memory <= 8: # cmd_opts.medvram = True # default_offload_mode = "model" # log.info(f"Device detect: memory={gpu_memory:.1f} optimization=medvram") else: default_offload_mode = "balanced" - log.info(f"Device detect: memory={gpu_memory:.1f} optimization=balanced") + log.info(f"Device detect: memory={gpu_memory:.1f} default=balanced") elif cmd_opts.medvram: default_offload_mode = "balanced" elif cmd_opts.lowvram: From 4a2af7b037bb7767e87512ce1500fa369b4a930b Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Thu, 9 Jan 2025 11:12:39 -0500 Subject: [PATCH 235/249] pipeline restore args after batch and restore pipeline after base Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 4 +++- modules/control/run.py | 3 ++- modules/processing_args.py | 12 +++++++++--- modules/processing_diffusers.py | 2 +- modules/processing_helpers.py | 7 ++++++- modules/sd_models.py | 3 +++ 6 files changed, 24 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 640b1f300..300e49ecf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Change Log for SD.Next -## Update for 2025-01-08 +## Update for 2025-01-09 - [Allegro Video](https://huggingface.co/rhymes-ai/Allegro) - optimizations: full offload and quantization support @@ -53,6 +53,8 @@ - lora diffusers method apply only once - lora diffusers method set prompt tags and metadata - flux support on-the-fly quantization for bnb of unet only + - control restore pipeline before running hires + - restore args after batch run ## Update for 2024-12-31 diff --git a/modules/control/run.py b/modules/control/run.py index b96718e0d..93a9c43fc 100644 --- a/modules/control/run.py +++ b/modules/control/run.py @@ -188,7 +188,7 @@ def check_enabled(p, unit_type, units, active_model, active_strength, active_sta selected_models = None elif len(active_model) == 1: selected_models = active_model[0].model if active_model[0].model is not None else None - p.is_tile = p.is_tile or 'tile' in active_model[0].model_id.lower() + p.is_tile = p.is_tile or 'tile' in (active_model[0].model_id or '').lower() has_models = selected_models is not None control_conditioning = active_strength[0] if len(active_strength) > 0 else 1 # strength or list[strength] control_guidance_start = active_start[0] if len(active_start) > 0 else 0 @@ -687,6 +687,7 @@ def control_run(state: str = '', if pipe is not None: # run new pipeline if not hasattr(pipe, 'restore_pipeline') and video is None: pipe.restore_pipeline = restore_pipeline + shared.sd_model.restore_pipeline = restore_pipeline debug(f'Control exec pipeline: task={sd_models.get_diffusers_task(pipe)} class={pipe.__class__}') # debug(f'Control exec pipeline: p={vars(p)}') # debug(f'Control exec pipeline: args={p.task_args} image={p.task_args.get("image", None)} control={p.task_args.get("control_image", None)} mask={p.task_args.get("mask_image", None) or p.image_mask} ref={p.task_args.get("ref_image", None)}') diff --git a/modules/processing_args.py b/modules/processing_args.py index e73cda85c..62b0db5cf 100644 --- a/modules/processing_args.py +++ b/modules/processing_args.py @@ -1,6 +1,7 @@ import typing import os import re +import copy import math import time import inspect @@ -122,7 +123,7 @@ def set_pipeline_args(p, model, prompts:list, negative_prompts:list, prompts_2:t if debug_enabled: debug_log(f'Diffusers pipeline possible: {possible}') - prompts, negative_prompts, prompts_2, negative_prompts_2 = fix_prompts(prompts, negative_prompts, prompts_2, negative_prompts_2) + prompts, negative_prompts, prompts_2, negative_prompts_2 = fix_prompts(p, prompts, negative_prompts, prompts_2, negative_prompts_2) steps = kwargs.get("num_inference_steps", None) or len(getattr(p, 'timesteps', ['1'])) clip_skip = kwargs.pop("clip_skip", 1) @@ -278,7 +279,10 @@ def set_pipeline_args(p, model, prompts:list, negative_prompts:list, prompts_2:t args['callback'] = diffusers_callback_legacy if 'image' in kwargs: - p.init_images = kwargs['image'] if isinstance(kwargs['image'], list) else [kwargs['image']] + if isinstance(kwargs['image'], list) and isinstance(kwargs['image'][0], Image.Image): + p.init_images = kwargs['image'] + if isinstance(kwargs['image'], Image.Image): + p.init_images = [kwargs['image']] # handle remaining args for arg in kwargs: @@ -360,4 +364,6 @@ def set_pipeline_args(p, model, prompts:list, negative_prompts:list, prompts_2:t shared.log.debug(f'Profile: pipeline args: {t1-t0:.2f}') if debug_enabled: debug_log(f'Diffusers pipeline args: {args}') - return args + + _args = copy.deepcopy(args) # pipeline may modify underlying args + return _args diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index a21712fae..ea2cf56ea 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -189,7 +189,7 @@ def process_hires(p: processing.StableDiffusionProcessing, output): # hires if p.hr_force and strength == 0: - shared.log.warning('HiRes skip: denoising=0') + shared.log.warning('Hires skip: denoising=0') p.hr_force = False if p.hr_force: shared.sd_model = sd_models.set_diffuser_pipe(shared.sd_model, sd_models.DiffusersTaskType.IMAGE_2_IMAGE) diff --git a/modules/processing_helpers.py b/modules/processing_helpers.py index e4a35442b..93896a02f 100644 --- a/modules/processing_helpers.py +++ b/modules/processing_helpers.py @@ -428,11 +428,16 @@ def resize_hires(p, latents): # input=latents output=pil if not latent_upscaler return resized_images -def fix_prompts(prompts, negative_prompts, prompts_2, negative_prompts_2): +def fix_prompts(p, prompts, negative_prompts, prompts_2, negative_prompts_2): if type(prompts) is str: prompts = [prompts] if type(negative_prompts) is str: negative_prompts = [negative_prompts] + if hasattr(p, '[init_images]') and p.init_images is not None and len(p.init_images) > 1: + while len(prompts) < len(p.init_images): + prompts.append(prompts[-1]) + while len(negative_prompts) < len(p.init_images): + negative_prompts.append(negative_prompts[-1]) while len(negative_prompts) < len(prompts): negative_prompts.append(negative_prompts[-1]) while len(prompts) < len(negative_prompts): diff --git a/modules/sd_models.py b/modules/sd_models.py index 1916fa7ec..ff64db644 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -1243,6 +1243,7 @@ def set_diffuser_pipe(pipe, new_pipe_type): image_encoder = getattr(pipe, "image_encoder", None) feature_extractor = getattr(pipe, "feature_extractor", None) mask_processor = getattr(pipe, "mask_processor", None) + restore_pipeline = getattr(pipe, "restore_pipeline", None) if new_pipe is None: if hasattr(pipe, 'config'): # real pipeline which can be auto-switched @@ -1292,6 +1293,8 @@ def set_diffuser_pipe(pipe, new_pipe_type): new_pipe.feature_extractor = feature_extractor if mask_processor is not None: new_pipe.mask_processor = mask_processor + if restore_pipeline is not None: + new_pipe.restore_pipeline = restore_pipeline if new_pipe.__class__.__name__ in ['FluxPipeline', 'StableDiffusion3Pipeline']: new_pipe.register_modules(image_encoder = image_encoder) new_pipe.register_modules(feature_extractor = feature_extractor) From e9cf6a57acedd5724186f9f7c5f6b78ae73892ce Mon Sep 17 00:00:00 2001 From: Seunghoon Lee Date: Fri, 10 Jan 2025 14:30:42 +0900 Subject: [PATCH 236/249] ZLUDA 3.8.6 --- modules/zluda_installer.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/modules/zluda_installer.py b/modules/zluda_installer.py index 89dac35f2..3593f2641 100644 --- a/modules/zluda_installer.py +++ b/modules/zluda_installer.py @@ -4,7 +4,6 @@ import ctypes import shutil import zipfile -import subprocess import urllib.request from typing import Optional, Union from modules import rocm @@ -17,7 +16,6 @@ } HIPSDK_TARGETS = ['rocblas.dll', 'rocsolver.dll'] ZLUDA_TARGETS = ('nvcuda.dll', 'nvml.dll',) -version: Union[str, None] = None experimental_hipBLASLt_support: bool = False default_agent: Union[rocm.Agent, None] = None @@ -32,7 +30,6 @@ def set_default_agent(agent: rocm.Agent): def install(zluda_path: os.PathLike) -> None: - if os.path.exists(zluda_path): __initialize(zluda_path) return @@ -103,14 +100,8 @@ def get_default_torch_version(agent: Optional[rocm.Agent]) -> str: def __initialize(zluda_path: os.PathLike): - global version, experimental_hipBLASLt_support # pylint: disable=global-statement - - if version is not None: - return - - process = subprocess.run(["zluda", "--version"], cwd=zluda_path, shell=True, check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - version = process.stdout.decode(encoding="utf8", errors="ignore")[6:].strip() - experimental_hipBLASLt_support = version == "3.8.5.pre1" + global experimental_hipBLASLt_support # pylint: disable=global-statement + experimental_hipBLASLt_support = os.path.exists(os.path.join(zluda_path, 'cublasLt.dll')) if experimental_hipBLASLt_support: HIPSDK_TARGETS.append('hipblaslt.dll') From 2066652c7380c1206a76bd685e2dae3d8d0e5da2 Mon Sep 17 00:00:00 2001 From: Seunghoon Lee Date: Fri, 10 Jan 2025 15:52:07 +0900 Subject: [PATCH 237/249] add zluda nightly flag --- modules/zluda_installer.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/modules/zluda_installer.py b/modules/zluda_installer.py index 3593f2641..a3cc6d7ab 100644 --- a/modules/zluda_installer.py +++ b/modules/zluda_installer.py @@ -34,10 +34,11 @@ def install(zluda_path: os.PathLike) -> None: __initialize(zluda_path) return - commit = os.environ.get("ZLUDA_HASH", "1b6e012d8f2404840b524e2abae12cb91e1ac01d") - if rocm.version == "6.1": - commit = "c0804ca624963aab420cb418412b1c7fbae3454b" - urllib.request.urlretrieve(f'https://github.com/lshqqytiger/ZLUDA/releases/download/rel.{commit}/ZLUDA-windows-rocm{rocm.version[0]}-amd64.zip', '_zluda') + platform = "windows" + commit = os.environ.get("ZLUDA_HASH", "d60bddbc870827566b3d2d417e00e1d2d8acc026") + if os.environ.get("ZLUDA_NIGHTLY", "0") == "1": + platform = "nightly-" + platform + urllib.request.urlretrieve(f'https://github.com/lshqqytiger/ZLUDA/releases/download/rel.{commit}/ZLUDA-{platform}-rocm{rocm.version[0]}-amd64.zip', '_zluda') with zipfile.ZipFile('_zluda', 'r') as archive: infos = archive.infolist() for info in infos: From 5baba50a59a214088fc0b7674bd5f8a6d4222a29 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 10 Jan 2025 08:00:17 -0500 Subject: [PATCH 238/249] fix pulid Signed-off-by: Vladimir Mandic --- modules/control/run.py | 4 +++- modules/processing_args.py | 4 +++- modules/processing_diffusers.py | 2 +- modules/sd_models.py | 2 ++ 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/modules/control/run.py b/modules/control/run.py index 93a9c43fc..74928dc64 100644 --- a/modules/control/run.py +++ b/modules/control/run.py @@ -410,7 +410,9 @@ def control_run(state: str = '', blended_image = None # set pipeline - if pipe.__class__.__name__ != shared.sd_model.__class__.__name__: + if pipe is None: + return [], '', '', 'Pipeline not set' + elif pipe.__class__.__name__ != shared.sd_model.__class__.__name__: original_pipeline = shared.sd_model shared.sd_model = pipe sd_models.move_model(shared.sd_model, shared.device) diff --git a/modules/processing_args.py b/modules/processing_args.py index 62b0db5cf..b703a80fa 100644 --- a/modules/processing_args.py +++ b/modules/processing_args.py @@ -365,5 +365,7 @@ def set_pipeline_args(p, model, prompts:list, negative_prompts:list, prompts_2:t if debug_enabled: debug_log(f'Diffusers pipeline args: {args}') - _args = copy.deepcopy(args) # pipeline may modify underlying args + _args = {} + for k, v in args.items(): + _args[k] = copy.deepcopy(v) if not torch.is_tensor(v) else v # pipeline may modify underlying args return _args diff --git a/modules/processing_diffusers.py b/modules/processing_diffusers.py index ea2cf56ea..c20ba85a8 100644 --- a/modules/processing_diffusers.py +++ b/modules/processing_diffusers.py @@ -175,7 +175,7 @@ def process_hires(p: processing.StableDiffusionProcessing, output): if shared.opts.samples_save and not p.do_not_save_samples and shared.opts.save_images_before_highres_fix and hasattr(shared.sd_model, 'vae'): save_intermediate(p, latents=output.images, suffix="-before-hires") shared.state.update('Upscale', 0, 1) - output.images = resize_hires(p, latents=output.images) + output.images = resize_hires(p, latents=output.images) if output is not None else [] sd_hijack_hypertile.hypertile_set(p, hr=True) latent_upscale = shared.latent_upscale_modes.get(p.hr_upscaler, None) diff --git a/modules/sd_models.py b/modules/sd_models.py index ff64db644..9cbdc67dc 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -1085,6 +1085,8 @@ def get_signature(cls): def get_call(cls): + if cls is None: + return [] signature = inspect.signature(cls.__call__, follow_wrapped=True, eval_str=True) return signature.parameters From ee50aea34bea18364172155c194c9df326455eea Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 10 Jan 2025 08:16:08 -0500 Subject: [PATCH 239/249] control processor handle null output Signed-off-by: Vladimir Mandic --- modules/control/proc/dwpose/__init__.py | 14 +++++++++++--- modules/control/processors.py | 3 +++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/modules/control/proc/dwpose/__init__.py b/modules/control/proc/dwpose/__init__.py index 4469e10c8..d8fdb9618 100644 --- a/modules/control/proc/dwpose/__init__.py +++ b/modules/control/proc/dwpose/__init__.py @@ -13,12 +13,14 @@ from modules.control.util import HWC3, resize_image from .draw import draw_bodypose, draw_handpose, draw_facepose checked_ok = False +busy = False def check_dependencies(): - global checked_ok # pylint: disable=global-statement + global checked_ok, busy # pylint: disable=global-statement debug = log.trace if os.environ.get('SD_DWPOSE_DEBUG', None) is not None else lambda *args, **kwargs: None packages = [ + 'termcolor', 'openmim==0.3.9', 'mmengine==0.10.4', 'mmcv==2.1.0', @@ -68,8 +70,13 @@ def __init__(self, det_config=None, det_ckpt=None, pose_config=None, pose_ckpt=N if not checked_ok: if not check_dependencies(): return - from .wholebody import Wholebody - self.pose_estimation = Wholebody(det_config, det_ckpt, pose_config, pose_ckpt, device) + Wholebody = None + try: + from .wholebody import Wholebody + except Exception as e: + log.error(f'DWPose: {e}') + if Wholebody is not None: + self.pose_estimation = Wholebody(det_config, det_ckpt, pose_config, pose_ckpt, device) def to(self, device): self.pose_estimation.to(device) @@ -78,6 +85,7 @@ def to(self, device): def __call__(self, input_image, detect_resolution=512, image_resolution=512, output_type="pil", min_confidence=0.3, **kwargs): if self.pose_estimation is None: log.error("DWPose: not loaded") + return None input_image = cv2.cvtColor(np.array(input_image, dtype=np.uint8), cv2.COLOR_RGB2BGR) input_image = HWC3(input_image) diff --git a/modules/control/processors.py b/modules/control/processors.py index 2f15c2d3f..aa1275d62 100644 --- a/modules/control/processors.py +++ b/modules/control/processors.py @@ -253,6 +253,9 @@ def __call__(self, image_input: Image, mode: str = 'RGB', resize_mode: int = 0, image_resized = image_input with devices.inference_context(): image_process = self.model(image_resized, **kwargs) + if image_process is None: + log.error(f'Control Processor: id="{self.processor_id}" no image') + return image_input if isinstance(image_process, np.ndarray): if np.max(image_process) < 2: image_process = (255.0 * image_process).astype(np.uint8) From 1516aef16b7a2fe10cb0cfb9c95ff0e1d2aa56b6 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 10 Jan 2025 08:20:42 -0500 Subject: [PATCH 240/249] update wiki Signed-off-by: Vladimir Mandic --- wiki | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wiki b/wiki index ba50d70e4..7c2400e9d 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit ba50d70e474d0ff06fe3ac24c18c192bdf210359 +Subproject commit 7c2400e9dc5dee3c52eac6bbfa88352f7815454a From a4d9d7f32575523b213c48163375cdc7062262a7 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 10 Jan 2025 08:25:00 -0500 Subject: [PATCH 241/249] cleanup Signed-off-by: Vladimir Mandic --- installer.py | 1 - modules/instantir/ip_adapter/ip_adapter.py | 2 -- modules/linfusion/linfusion.py | 4 +--- modules/lora/lora_convert.py | 2 -- modules/meissonic/transformer.py | 4 ---- modules/omnigen/model.py | 1 - modules/pixelsmith/vae.py | 4 ---- modules/pulid/eva_clip/hf_model.py | 1 - modules/schedulers/scheduler_dc.py | 1 - modules/schedulers/scheduler_tdd.py | 4 +--- modules/schedulers/scheduler_vdm.py | 1 - modules/todo/todo_merge.py | 1 - 12 files changed, 2 insertions(+), 24 deletions(-) diff --git a/installer.py b/installer.py index 96259da36..8f7ed7aca 100644 --- a/installer.py +++ b/installer.py @@ -1010,7 +1010,6 @@ def ensure_base_requirements(): setuptools_version = '69.5.1' def update_setuptools(): - # print('Install base requirements') global pkg_resources, setuptools, distutils # pylint: disable=global-statement # python may ship with incompatible setuptools subprocess.run(f'"{sys.executable}" -m pip install setuptools=={setuptools_version}', shell=True, check=False, env=os.environ, stdout=subprocess.PIPE, stderr=subprocess.PIPE) diff --git a/modules/instantir/ip_adapter/ip_adapter.py b/modules/instantir/ip_adapter/ip_adapter.py index 10f01d4f3..7f4bcbc45 100644 --- a/modules/instantir/ip_adapter/ip_adapter.py +++ b/modules/instantir/ip_adapter/ip_adapter.py @@ -169,8 +169,6 @@ def load_from_checkpoint(self, ckpt_path: str): if "latents" in state_dict["image_proj"] and "latents" in self.image_proj.state_dict(): # Check if the shapes are mismatched if state_dict["image_proj"]["latents"].shape != self.image_proj.state_dict()["latents"].shape: - print(f"Shapes of 'image_proj.latents' in checkpoint {ckpt_path} and current model do not match.") - print("Removing 'latents' from checkpoint and loading the rest of the weights.") del state_dict["image_proj"]["latents"] strict_load_image_proj_model = False diff --git a/modules/linfusion/linfusion.py b/modules/linfusion/linfusion.py index 724cf2f3f..d8317e39b 100644 --- a/modules/linfusion/linfusion.py +++ b/modules/linfusion/linfusion.py @@ -89,9 +89,7 @@ def construct_for( pipe_name_path = pipe_name_path or pipeline._internal_dict._name_or_path # pylint: disable=protected-access pretrained_model_name_or_path = model_dict.get(pipe_name_path, None) if pretrained_model_name_or_path: - print( - f"Matching LinFusion '{pretrained_model_name_or_path}' for pipeline '{pipe_name_path}'." - ) + pass else: raise RuntimeError( f"LinFusion not found for pipeline [{pipe_name_path}], please provide the path." diff --git a/modules/lora/lora_convert.py b/modules/lora/lora_convert.py index 032ffa5a3..c2685aacd 100644 --- a/modules/lora/lora_convert.py +++ b/modules/lora/lora_convert.py @@ -205,8 +205,6 @@ def _convert_to_ai_toolkit_cat(sds_sd, ait_sd, sds_key, ait_keys, dims=None): up_weight[i : i + dims[j], k * ait_rank : (k + 1) * ait_rank] == 0 ) i += dims[j] - # if is_sparse: - # print(f"weight is sparse: {sds_key}") # make ai-toolkit weight ait_down_keys = [k + ".lora_down.weight" for k in ait_keys] diff --git a/modules/meissonic/transformer.py b/modules/meissonic/transformer.py index 43e77ddc7..543c30108 100644 --- a/modules/meissonic/transformer.py +++ b/modules/meissonic/transformer.py @@ -670,15 +670,11 @@ def __init__( self.upsample = None def forward(self, x): - # print("before,", x.shape) if self.downsample is not None: - # print('downsample') x = self.downsample(x) if self.upsample is not None: - # print('upsample') x = self.upsample(x) - # print("after,", x.shape) return x diff --git a/modules/omnigen/model.py b/modules/omnigen/model.py index 3a42263d2..17d696b53 100644 --- a/modules/omnigen/model.py +++ b/modules/omnigen/model.py @@ -259,7 +259,6 @@ def cropped_pos_embed(self, height, width): left = (self.pos_embed_max_size - width) // 2 spatial_pos_embed = self.pos_embed.reshape(1, self.pos_embed_max_size, self.pos_embed_max_size, -1) spatial_pos_embed = spatial_pos_embed[:, top : top + height, left : left + width, :] - # print(top, top + height, left, left + width, spatial_pos_embed.size()) spatial_pos_embed = spatial_pos_embed.reshape(1, -1, spatial_pos_embed.shape[-1]) return spatial_pos_embed diff --git a/modules/pixelsmith/vae.py b/modules/pixelsmith/vae.py index 93d376cfd..98e051695 100644 --- a/modules/pixelsmith/vae.py +++ b/modules/pixelsmith/vae.py @@ -681,10 +681,6 @@ def __init__( if self.unknown_index == "extra": self.unknown_index = self.re_embed self.re_embed = self.re_embed + 1 - print( - f"Remapping {self.n_e} indices to {self.re_embed} indices. " - f"Using {self.unknown_index} for unknown indices." - ) else: self.re_embed = n_e diff --git a/modules/pulid/eva_clip/hf_model.py b/modules/pulid/eva_clip/hf_model.py index d148bbff2..0b9551993 100644 --- a/modules/pulid/eva_clip/hf_model.py +++ b/modules/pulid/eva_clip/hf_model.py @@ -222,7 +222,6 @@ def lock(self, unlocked_layers:int=0, freeze_layer_norm:bool=True): encoder = self.transformer.encoder if hasattr(self.transformer, 'encoder') else self.transformer layer_list = getattr(encoder, arch_dict[self.config.model_type]["config_names"]["layer_attr"]) - print(f"Unlocking {unlocked_layers}/{len(layer_list) + 1} layers of hf model") embeddings = getattr( self.transformer, arch_dict[self.config.model_type]["config_names"]["token_embeddings_attr"]) modules = [embeddings, *layer_list][:-unlocked_layers] diff --git a/modules/schedulers/scheduler_dc.py b/modules/schedulers/scheduler_dc.py index a1ccfbeba..190588855 100644 --- a/modules/schedulers/scheduler_dc.py +++ b/modules/schedulers/scheduler_dc.py @@ -820,7 +820,6 @@ def closure(ratio_param): loss.backward() optimizer.step() ratio_bound = bound_func(ratio_param) - print(f'iter [{iter_}]', ratio_bound.item(), loss.item()) torch.cuda.empty_cache() return ratio_bound.data.detach().item() diff --git a/modules/schedulers/scheduler_tdd.py b/modules/schedulers/scheduler_tdd.py index 03c9da1a5..125ef1b3b 100644 --- a/modules/schedulers/scheduler_tdd.py +++ b/modules/schedulers/scheduler_tdd.py @@ -117,8 +117,7 @@ def set_timesteps(self, num_inference_steps: int, device: Union[str, torch.devic timesteps = tcd_origin_timesteps[inference_indices] if self.special_jump: if self.tdd_train_step == 50: - #timesteps = np.array([999., 879., 759., 499., 259.]) - print(timesteps) + pass elif self.tdd_train_step == 250: if num_inference_steps == 5: timesteps = np.array([999., 875., 751., 499., 251.]) @@ -203,7 +202,6 @@ def set_timesteps_s(self, eta: float = 0.0): sigmas_s = np.array(((1 - self.alphas_cumprod) / self.alphas_cumprod) ** 0.5) if self.config.use_karras_sigmas: - print("have not write") pass else: sigmas_s = np.interp(timesteps_s, np.arange(0, len(sigmas_s)), sigmas_s) diff --git a/modules/schedulers/scheduler_vdm.py b/modules/schedulers/scheduler_vdm.py index 492c30a0c..4f48db163 100644 --- a/modules/schedulers/scheduler_vdm.py +++ b/modules/schedulers/scheduler_vdm.py @@ -355,7 +355,6 @@ def step( ) # 3. Clip or threshold "predicted x_0" - # print({ 'timestep': timestep.item(), 'min': pred_original_sample.min().item(), 'max': pred_original_sample.max().item(), 'alpha': alpha.item(), 'sigma': sigma.item() }) if self.config.thresholding: pred_original_sample = self._threshold_sample(pred_original_sample) elif self.config.clip_sample: diff --git a/modules/todo/todo_merge.py b/modules/todo/todo_merge.py index bfbff5621..77840d6ed 100644 --- a/modules/todo/todo_merge.py +++ b/modules/todo/todo_merge.py @@ -25,7 +25,6 @@ def init_generator(device: torch.device, fallback: torch.Generator = None): """ Forks the current default random generator given device. """ - print(f"init_generator device = {device}") if device.type == "cpu": return torch.Generator(device="cpu").set_state(torch.get_rng_state()) elif device.type == "cuda": From 954e09e7872d9a761f8dfa9dac272edd5d00e94b Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 10 Jan 2025 10:00:57 -0500 Subject: [PATCH 242/249] reduce control logging Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 2 +- modules/control/processors.py | 4 ++-- modules/control/units/controlnet.py | 2 +- modules/control/units/lite.py | 2 +- modules/control/units/t2iadapter.py | 2 +- modules/control/units/xs.py | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 300e49ecf..1b4ffb88b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Change Log for SD.Next -## Update for 2025-01-09 +## Update for 2025-01-10 - [Allegro Video](https://huggingface.co/rhymes-ai/Allegro) - optimizations: full offload and quantization support diff --git a/modules/control/processors.py b/modules/control/processors.py index aa1275d62..38b5f2062 100644 --- a/modules/control/processors.py +++ b/modules/control/processors.py @@ -167,13 +167,13 @@ def load(self, processor_id: str = None, force: bool = True) -> str: self.config(processor_id) else: if not force and self.model is not None: - log.debug(f'Control Processor: id={processor_id} already loaded') + # log.debug(f'Control Processor: id={processor_id} already loaded') return '' if processor_id not in config: log.error(f'Control Processor unknown: id="{processor_id}" available={list(config)}') return f'Processor failed to load: {processor_id}' cls = config[processor_id]['class'] - log.debug(f'Control Processor loading: id="{processor_id}" class={cls.__name__}') + # log.debug(f'Control Processor loading: id="{processor_id}" class={cls.__name__}') debug(f'Control Processor config={self.load_config}') if 'DWPose' in processor_id: det_ckpt = 'https://download.openmmlab.com/mmdetection/v2.0/yolox/yolox_l_8x8_300e_coco/yolox_l_8x8_300e_coco_20211126_140236-d3bd2b23.pth' diff --git a/modules/control/units/controlnet.py b/modules/control/units/controlnet.py index c887aca8f..d6175ead0 100644 --- a/modules/control/units/controlnet.py +++ b/modules/control/units/controlnet.py @@ -255,7 +255,7 @@ def load(self, model_id: str = None, force: bool = True) -> str: self.model = model_path return if model_id == self.model_id and not force: - log.debug(f'Control {what} model: id="{model_id}" path="{model_path}" already loaded') + # log.debug(f'Control {what} model: id="{model_id}" path="{model_path}" already loaded') return log.debug(f'Control {what} model loading: id="{model_id}" path="{model_path}"') cls, _config = self.get_class(model_id) diff --git a/modules/control/units/lite.py b/modules/control/units/lite.py index 0c10d8d53..914f56c9f 100644 --- a/modules/control/units/lite.py +++ b/modules/control/units/lite.py @@ -95,7 +95,7 @@ def load(self, model_id: str = None, force: bool = True) -> str: log.error(f'Control {what} model load failed: id="{model_id}" error=unknown model id') return if model_id == self.model_id and not force: - log.debug(f'Control {what} model: id="{model_id}" path="{model_path}" already loaded') + # log.debug(f'Control {what} model: id="{model_id}" path="{model_path}" already loaded') return log.debug(f'Control {what} model loading: id="{model_id}" path="{model_path}" {self.load_config}') if model_path.endswith('.safetensors'): diff --git a/modules/control/units/t2iadapter.py b/modules/control/units/t2iadapter.py index 6e15abe3d..6666f67ce 100644 --- a/modules/control/units/t2iadapter.py +++ b/modules/control/units/t2iadapter.py @@ -102,7 +102,7 @@ def load(self, model_id: str = None, force: bool = True) -> str: log.error(f'Control {what} model load failed: id="{model_id}" error=unknown model id') return if model_id == self.model_id and not force: - log.debug(f'Control {what} model: id="{model_id}" path="{model_path}" already loaded') + # log.debug(f'Control {what} model: id="{model_id}" path="{model_path}" already loaded') return log.debug(f'Control {what} model loading: id="{model_id}" path="{model_path}"') if model_path.endswith('.pth') or model_path.endswith('.pt') or model_path.endswith('.safetensors') or model_path.endswith('.bin'): diff --git a/modules/control/units/xs.py b/modules/control/units/xs.py index ff21a5ed7..a90cd97d2 100644 --- a/modules/control/units/xs.py +++ b/modules/control/units/xs.py @@ -91,7 +91,7 @@ def load(self, model_id: str = None, time_embedding_mix: float = 0.0, force: boo log.error(f'Control {what} model load failed: id="{model_id}" error=unknown model id') return if model_id == self.model_id and not force: - log.debug(f'Control {what} model: id="{model_id}" path="{model_path}" already loaded') + # log.debug(f'Control {what} model: id="{model_id}" path="{model_path}" already loaded') return self.load_config['time_embedding_mix'] = time_embedding_mix log.debug(f'Control {what} model loading: id="{model_id}" path="{model_path}" {self.load_config}') From df5158705491693f4791481cda0c4fb10ece4635 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 10 Jan 2025 10:07:24 -0500 Subject: [PATCH 243/249] fix splash on mobile Signed-off-by: Vladimir Mandic --- javascript/base.css | 2 +- javascript/sdnext.css | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/javascript/base.css b/javascript/base.css index a30a71845..cadfd1868 100644 --- a/javascript/base.css +++ b/javascript/base.css @@ -129,7 +129,7 @@ div:has(>#tab-browser-folders) { flex-grow: 0 !important; background-color: var( /* loader */ .splash { position: fixed; top: 0; left: 0; width: 100vw; height: 100vh; z-index: 1000; display: block; text-align: center; } .motd { margin-top: 2em; color: var(--body-text-color-subdued); font-family: monospace; font-variant: all-petite-caps; } -.splash-img { margin: 10% auto 0 auto; width: 512px; background-repeat: no-repeat; height: 512px; animation: color 10s infinite alternate; } +.splash-img { margin: 10% auto 0 auto; width: 512px; background-repeat: no-repeat; height: 512px; animation: color 10s infinite alternate; max-width: 80vw; background-size: contain; } .loading { color: white; position: absolute; top: 20%; left: 50%; transform: translateX(-50%); } .loader { width: 300px; height: 300px; border: var(--spacing-md) solid transparent; border-radius: 50%; border-top: var(--spacing-md) solid var(--primary-600); animation: spin 4s linear infinite; position: relative; } .loader::before, .loader::after { content: ""; position: absolute; top: 6px; bottom: 6px; left: 6px; right: 6px; border-radius: 50%; border: var(--spacing-md) solid transparent; } diff --git a/javascript/sdnext.css b/javascript/sdnext.css index 2e85c6988..91ab6318d 100644 --- a/javascript/sdnext.css +++ b/javascript/sdnext.css @@ -346,7 +346,7 @@ div:has(>#tab-gallery-folders) { flex-grow: 0 !important; background-color: var( /* loader */ .splash { position: fixed; top: 0; left: 0; width: 100vw; height: 100vh; z-index: 1000; display: block; text-align: center; } .motd { margin-top: 2em; color: var(--body-text-color-subdued); font-family: monospace; font-variant: all-petite-caps; } -.splash-img { margin: 10% auto 0 auto; width: 512px; background-repeat: no-repeat; height: 512px; animation: color 10s infinite alternate; } +.splash-img { margin: 10% auto 0 auto; width: 512px; background-repeat: no-repeat; height: 512px; animation: color 10s infinite alternate; max-width: 80vw; background-size: contain; } .loading { color: white; position: absolute; top: 20%; left: 50%; transform: translateX(-50%); } .loader { width: 300px; height: 300px; border: var(--spacing-md) solid transparent; border-radius: 50%; border-top: var(--spacing-md) solid var(--primary-600); animation: spin 4s linear infinite; position: relative; } .loader::before, .loader::after { content: ""; position: absolute; top: 6px; bottom: 6px; left: 6px; right: 6px; border-radius: 50%; border: var(--spacing-md) solid transparent; } From f7db34edd48c2e8f448723a28c55ce2eb51b2371 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 10 Jan 2025 12:18:33 -0500 Subject: [PATCH 244/249] controlnet load lock Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 1 + modules/control/units/controlnet.py | 135 ++++++++++++++-------------- 2 files changed, 70 insertions(+), 66 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b4ffb88b..cc71882b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,6 +55,7 @@ - flux support on-the-fly quantization for bnb of unet only - control restore pipeline before running hires - restore args after batch run + - control add load lock ## Update for 2024-12-31 diff --git a/modules/control/units/controlnet.py b/modules/control/units/controlnet.py index d6175ead0..0e805eb0c 100644 --- a/modules/control/units/controlnet.py +++ b/modules/control/units/controlnet.py @@ -1,5 +1,6 @@ import os import time +import threading from typing import Union from diffusers import StableDiffusionPipeline, StableDiffusionXLPipeline, FluxPipeline, StableDiffusion3Pipeline, ControlNetModel from modules.control.units import detect @@ -116,6 +117,7 @@ all_models.update(predefined_f1) all_models.update(predefined_sd3) cache_dir = 'models/control/controlnet' +load_lock = threading.Lock() def find_models(): @@ -236,73 +238,74 @@ def load_safetensors(self, model_id, model_path): self.model = cls.from_single_file(model_path, config=config, **self.load_config) def load(self, model_id: str = None, force: bool = True) -> str: - try: - t0 = time.time() - model_id = model_id or self.model_id - if model_id is None or model_id == 'None': - self.reset() - return - if model_id not in all_models: - log.error(f'Control {what} unknown model: id="{model_id}" available={list(all_models)}') - return - model_path = all_models[model_id] - if model_path == '': - return - if model_path is None: - log.error(f'Control {what} model load failed: id="{model_id}" error=unknown model id') - return - if 'lora' in model_id.lower(): - self.model = model_path - return - if model_id == self.model_id and not force: - # log.debug(f'Control {what} model: id="{model_id}" path="{model_path}" already loaded') - return - log.debug(f'Control {what} model loading: id="{model_id}" path="{model_path}"') - cls, _config = self.get_class(model_id) - if model_path.endswith('.safetensors'): - self.load_safetensors(model_id, model_path) - else: - kwargs = {} - if '/bin' in model_path: - model_path = model_path.replace('/bin', '') - self.load_config['use_safetensors'] = False - if cls is None: - log.error(f'Control {what} model load failed: id="{model_id}" unknown base model') + with load_lock: + try: + t0 = time.time() + model_id = model_id or self.model_id + if model_id is None or model_id == 'None': + self.reset() return - if variants.get(model_id, None) is not None: - kwargs['variant'] = variants[model_id] - self.model = cls.from_pretrained(model_path, **self.load_config, **kwargs) - if self.model is None: - return - if self.dtype is not None: - self.model.to(self.dtype) - if "ControlNet" in opts.nncf_compress_weights: - try: - log.debug(f'Control {what} model NNCF Compress: id="{model_id}"') - from installer import install - install('nncf==2.7.0', quiet=True) - from modules.sd_models_compile import nncf_compress_model - self.model = nncf_compress_model(self.model) - except Exception as e: - log.error(f'Control {what} model NNCF Compression failed: id="{model_id}" error={e}') - elif "ControlNet" in opts.optimum_quanto_weights: - try: - log.debug(f'Control {what} model Optimum Quanto: id="{model_id}"') - model_quant.load_quanto('Load model: type=ControlNet') - from modules.sd_models_compile import optimum_quanto_model - self.model = optimum_quanto_model(self.model) - except Exception as e: - log.error(f'Control {what} model Optimum Quanto failed: id="{model_id}" error={e}') - if self.device is not None: - self.model.to(self.device) - t1 = time.time() - self.model_id = model_id - log.debug(f'Control {what} model loaded: id="{model_id}" path="{model_path}" cls={cls.__name__} time={t1-t0:.2f}') - return f'{what} loaded model: {model_id}' - except Exception as e: - log.error(f'Control {what} model load failed: id="{model_id}" error={e}') - errors.display(e, f'Control {what} load') - return f'{what} failed to load model: {model_id}' + if model_id not in all_models: + log.error(f'Control {what} unknown model: id="{model_id}" available={list(all_models)}') + return + model_path = all_models[model_id] + if model_path == '': + return + if model_path is None: + log.error(f'Control {what} model load failed: id="{model_id}" error=unknown model id') + return + if 'lora' in model_id.lower(): + self.model = model_path + return + if model_id == self.model_id and not force: + # log.debug(f'Control {what} model: id="{model_id}" path="{model_path}" already loaded') + return + log.debug(f'Control {what} model loading: id="{model_id}" path="{model_path}"') + cls, _config = self.get_class(model_id) + if model_path.endswith('.safetensors'): + self.load_safetensors(model_id, model_path) + else: + kwargs = {} + if '/bin' in model_path: + model_path = model_path.replace('/bin', '') + self.load_config['use_safetensors'] = False + if cls is None: + log.error(f'Control {what} model load failed: id="{model_id}" unknown base model') + return + if variants.get(model_id, None) is not None: + kwargs['variant'] = variants[model_id] + self.model = cls.from_pretrained(model_path, **self.load_config, **kwargs) + if self.model is None: + return + if self.dtype is not None: + self.model.to(self.dtype) + if "ControlNet" in opts.nncf_compress_weights: + try: + log.debug(f'Control {what} model NNCF Compress: id="{model_id}"') + from installer import install + install('nncf==2.7.0', quiet=True) + from modules.sd_models_compile import nncf_compress_model + self.model = nncf_compress_model(self.model) + except Exception as e: + log.error(f'Control {what} model NNCF Compression failed: id="{model_id}" error={e}') + elif "ControlNet" in opts.optimum_quanto_weights: + try: + log.debug(f'Control {what} model Optimum Quanto: id="{model_id}"') + model_quant.load_quanto('Load model: type=ControlNet') + from modules.sd_models_compile import optimum_quanto_model + self.model = optimum_quanto_model(self.model) + except Exception as e: + log.error(f'Control {what} model Optimum Quanto failed: id="{model_id}" error={e}') + if self.device is not None: + self.model.to(self.device) + t1 = time.time() + self.model_id = model_id + log.info(f'Control {what} model loaded: id="{model_id}" path="{model_path}" cls={cls.__name__} time={t1-t0:.2f}') + return f'{what} loaded model: {model_id}' + except Exception as e: + log.error(f'Control {what} model load failed: id="{model_id}" error={e}') + errors.display(e, f'Control {what} load') + return f'{what} failed to load model: {model_id}' class ControlNetPipeline(): From 1fc72b6fb8b092bc2a231f4f6d321f3e9b643c31 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 10 Jan 2025 13:21:09 -0500 Subject: [PATCH 245/249] load thread locks Signed-off-by: Vladimir Mandic --- CHANGELOG.md | 2 +- installer.py | 11 +++- modules/cmd_args.py | 8 ++- modules/control/units/lite.py | 79 ++++++++++++++-------------- modules/control/units/t2iadapter.py | 81 +++++++++++++++-------------- modules/control/units/xs.py | 75 +++++++++++++------------- modules/deepbooru.py | 31 ++++++----- modules/interrogate.py | 55 +++++++++++--------- modules/paths.py | 8 +-- modules/postprocess/yolo.py | 47 +++++++++-------- modules/sd_checkpoint.py | 4 +- package.json | 2 +- scripts/prompt_enhance.py | 13 +++-- 13 files changed, 228 insertions(+), 188 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc71882b6..3df1eec47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ - refactored progress monitoring, job updates and live preview - improved metadata save and restore - startup tracing and optimizations + - threading load locks on model loads - **Schedulers**: - [TDD](https://github.com/RedAIGC/Target-Driven-Distillation) new super-fast scheduler that can generate images in 4-8 steps recommended to use with [TDD LoRA](https://huggingface.co/RED-AIGC/TDD/tree/main) @@ -55,7 +56,6 @@ - flux support on-the-fly quantization for bnb of unet only - control restore pipeline before running hires - restore args after batch run - - control add load lock ## Update for 2024-12-31 diff --git a/installer.py b/installer.py index 8f7ed7aca..7b351c799 100644 --- a/installer.py +++ b/installer.py @@ -1397,11 +1397,20 @@ def add_args(parser): group_log.add_argument('--docs', default=os.environ.get("SD_DOCS", False), action='store_true', help="Mount API docs, default: %(default)s") group_log.add_argument("--api-log", default=os.environ.get("SD_APILOG", True), action='store_true', help="Log all API requests") + group_nargs = parser.add_argument_group('Other') + group_nargs.add_argument('args', type=str, nargs='*') + def parse_args(parser): # command line args global args # pylint: disable=global-statement - args = parser.parse_args() + if "USED_VSCODE_COMMAND_PICKARGS" in os.environ: + import shlex + argv = shlex.split(" ".join(sys.argv[1:])) if "USED_VSCODE_COMMAND_PICKARGS" in os.environ else sys.argv[1:] + log.debug('VSCode Launch') + args = parser.parse_args(argv) + else: + args = parser.parse_args() return args diff --git a/modules/cmd_args.py b/modules/cmd_args.py index 018d802ee..54b888a17 100644 --- a/modules/cmd_args.py +++ b/modules/cmd_args.py @@ -1,4 +1,5 @@ import os +import sys import argparse from modules.paths import data_path @@ -154,7 +155,12 @@ def settings_args(opts, args): opts.onchange("lora_dir", lambda: setattr(args, "lora_dir", opts.lora_dir)) opts.onchange("lyco_dir", lambda: setattr(args, "lyco_dir", opts.lyco_dir)) - args = parser.parse_args() + if "USED_VSCODE_COMMAND_PICKARGS" in os.environ: + import shlex + argv = shlex.split(" ".join(sys.argv[1:])) if "USED_VSCODE_COMMAND_PICKARGS" in os.environ else sys.argv[1:] + args = parser.parse_args(argv) + else: + args = parser.parse_args() return args diff --git a/modules/control/units/lite.py b/modules/control/units/lite.py index 914f56c9f..854e5bd56 100644 --- a/modules/control/units/lite.py +++ b/modules/control/units/lite.py @@ -1,6 +1,7 @@ import os import time from typing import Union +import threading import numpy as np from PIL import Image from diffusers import StableDiffusionPipeline, StableDiffusionXLPipeline @@ -27,6 +28,7 @@ all_models.update(predefined_sd15) all_models.update(predefined_sdxl) cache_dir = 'models/control/lite' +load_lock = threading.Lock() def find_models(): @@ -79,44 +81,45 @@ def reset(self): self.model_id = None def load(self, model_id: str = None, force: bool = True) -> str: - try: - t0 = time.time() - model_id = model_id or self.model_id - if model_id is None or model_id == 'None': - self.reset() - return - if model_id not in all_models: - log.error(f'Control {what} unknown model: id="{model_id}" available={list(all_models)}') - return - model_path = all_models[model_id] - if model_path == '': - return - if model_path is None: - log.error(f'Control {what} model load failed: id="{model_id}" error=unknown model id') - return - if model_id == self.model_id and not force: - # log.debug(f'Control {what} model: id="{model_id}" path="{model_path}" already loaded') - return - log.debug(f'Control {what} model loading: id="{model_id}" path="{model_path}" {self.load_config}') - if model_path.endswith('.safetensors'): - self.model = ControlNetLLLite(model_path) - else: - import huggingface_hub as hf - folder, filename = os.path.split(model_path) - model_path = hf.hf_hub_download(repo_id=folder, filename=f'{filename}.safetensors', cache_dir=cache_dir) - self.model = ControlNetLLLite(model_path) - if self.device is not None: - self.model.to(self.device) - if self.dtype is not None: - self.model.to(self.dtype) - t1 = time.time() - self.model_id = model_id - log.debug(f'Control {what} model loaded: id="{model_id}" path="{model_path}" time={t1-t0:.2f}') - return f'{what} loaded model: {model_id}' - except Exception as e: - log.error(f'Control {what} model load failed: id="{model_id}" error={e}') - errors.display(e, f'Control {what} load') - return f'{what} failed to load model: {model_id}' + with load_lock: + try: + t0 = time.time() + model_id = model_id or self.model_id + if model_id is None or model_id == 'None': + self.reset() + return + if model_id not in all_models: + log.error(f'Control {what} unknown model: id="{model_id}" available={list(all_models)}') + return + model_path = all_models[model_id] + if model_path == '': + return + if model_path is None: + log.error(f'Control {what} model load failed: id="{model_id}" error=unknown model id') + return + if model_id == self.model_id and not force: + # log.debug(f'Control {what} model: id="{model_id}" path="{model_path}" already loaded') + return + log.debug(f'Control {what} model loading: id="{model_id}" path="{model_path}" {self.load_config}') + if model_path.endswith('.safetensors'): + self.model = ControlNetLLLite(model_path) + else: + import huggingface_hub as hf + folder, filename = os.path.split(model_path) + model_path = hf.hf_hub_download(repo_id=folder, filename=f'{filename}.safetensors', cache_dir=cache_dir) + self.model = ControlNetLLLite(model_path) + if self.device is not None: + self.model.to(self.device) + if self.dtype is not None: + self.model.to(self.dtype) + t1 = time.time() + self.model_id = model_id + log.debug(f'Control {what} model loaded: id="{model_id}" path="{model_path}" time={t1-t0:.2f}') + return f'{what} loaded model: {model_id}' + except Exception as e: + log.error(f'Control {what} model load failed: id="{model_id}" error={e}') + errors.display(e, f'Control {what} load') + return f'{what} failed to load model: {model_id}' class ControlLLitePipeline(): diff --git a/modules/control/units/t2iadapter.py b/modules/control/units/t2iadapter.py index 6666f67ce..66abdc75e 100644 --- a/modules/control/units/t2iadapter.py +++ b/modules/control/units/t2iadapter.py @@ -1,6 +1,7 @@ import os import time from typing import Union +import threading from diffusers import StableDiffusionPipeline, StableDiffusionXLPipeline, T2IAdapter, MultiAdapter, StableDiffusionAdapterPipeline, StableDiffusionXLAdapterPipeline # pylint: disable=unused-import from modules.shared import log from modules import errors, sd_models @@ -43,6 +44,7 @@ all_models.update(predefined_sd15) all_models.update(predefined_sdxl) cache_dir = 'models/control/adapter' +load_lock = threading.Lock() def list_models(refresh=False): @@ -87,45 +89,46 @@ def reset(self): self.model_id = None def load(self, model_id: str = None, force: bool = True) -> str: - try: - t0 = time.time() - model_id = model_id or self.model_id - if model_id is None or model_id == 'None': - self.reset() - return - if model_id not in all_models: - log.error(f'Control {what} unknown model: id="{model_id}" available={list(all_models)}') - return - model_path, model_args = all_models[model_id] - self.load_config.update(model_args) - if model_path is None: - log.error(f'Control {what} model load failed: id="{model_id}" error=unknown model id') - return - if model_id == self.model_id and not force: - # log.debug(f'Control {what} model: id="{model_id}" path="{model_path}" already loaded') - return - log.debug(f'Control {what} model loading: id="{model_id}" path="{model_path}"') - if model_path.endswith('.pth') or model_path.endswith('.pt') or model_path.endswith('.safetensors') or model_path.endswith('.bin'): - from huggingface_hub import hf_hub_download - parts = model_path.split('/') - repo_id = f'{parts[0]}/{parts[1]}' - filename = '/'.join(parts[2:]) - model = hf_hub_download(repo_id, filename, **self.load_config) - self.model = T2IAdapter.from_pretrained(model, **self.load_config) - else: - self.model = T2IAdapter.from_pretrained(model_path, **self.load_config) - if self.device is not None: - self.model.to(self.device) - if self.dtype is not None: - self.model.to(self.dtype) - t1 = time.time() - self.model_id = model_id - log.debug(f'Control {what} loaded: id="{model_id}" path="{model_path}" time={t1-t0:.2f}') - return f'{what} loaded model: {model_id}' - except Exception as e: - log.error(f'Control {what} model load failed: id="{model_id}" error={e}') - errors.display(e, f'Control {what} load') - return f'{what} failed to load model: {model_id}' + with load_lock: + try: + t0 = time.time() + model_id = model_id or self.model_id + if model_id is None or model_id == 'None': + self.reset() + return + if model_id not in all_models: + log.error(f'Control {what} unknown model: id="{model_id}" available={list(all_models)}') + return + model_path, model_args = all_models[model_id] + self.load_config.update(model_args) + if model_path is None: + log.error(f'Control {what} model load failed: id="{model_id}" error=unknown model id') + return + if model_id == self.model_id and not force: + # log.debug(f'Control {what} model: id="{model_id}" path="{model_path}" already loaded') + return + log.debug(f'Control {what} model loading: id="{model_id}" path="{model_path}"') + if model_path.endswith('.pth') or model_path.endswith('.pt') or model_path.endswith('.safetensors') or model_path.endswith('.bin'): + from huggingface_hub import hf_hub_download + parts = model_path.split('/') + repo_id = f'{parts[0]}/{parts[1]}' + filename = '/'.join(parts[2:]) + model = hf_hub_download(repo_id, filename, **self.load_config) + self.model = T2IAdapter.from_pretrained(model, **self.load_config) + else: + self.model = T2IAdapter.from_pretrained(model_path, **self.load_config) + if self.device is not None: + self.model.to(self.device) + if self.dtype is not None: + self.model.to(self.dtype) + t1 = time.time() + self.model_id = model_id + log.debug(f'Control {what} loaded: id="{model_id}" path="{model_path}" time={t1-t0:.2f}') + return f'{what} loaded model: {model_id}' + except Exception as e: + log.error(f'Control {what} model load failed: id="{model_id}" error={e}') + errors.display(e, f'Control {what} load') + return f'{what} failed to load model: {model_id}' class AdapterPipeline(): diff --git a/modules/control/units/xs.py b/modules/control/units/xs.py index a90cd97d2..232387582 100644 --- a/modules/control/units/xs.py +++ b/modules/control/units/xs.py @@ -1,6 +1,7 @@ import os import time from typing import Union +import threading from diffusers import StableDiffusionPipeline, StableDiffusionXLPipeline from modules.shared import log, opts, listdir from modules import errors, sd_models @@ -23,6 +24,7 @@ all_models.update(predefined_sd15) all_models.update(predefined_sdxl) cache_dir = 'models/control/xs' +load_lock = threading.Lock() def find_models(): @@ -75,42 +77,43 @@ def reset(self): self.model_id = None def load(self, model_id: str = None, time_embedding_mix: float = 0.0, force: bool = True) -> str: - try: - t0 = time.time() - model_id = model_id or self.model_id - if model_id is None or model_id == 'None': - self.reset() - return - if model_id not in all_models: - log.error(f'Control {what} unknown model: id="{model_id}" available={list(all_models)}') - return - model_path = all_models[model_id] - if model_path == '': - return - if model_path is None: - log.error(f'Control {what} model load failed: id="{model_id}" error=unknown model id') - return - if model_id == self.model_id and not force: - # log.debug(f'Control {what} model: id="{model_id}" path="{model_path}" already loaded') - return - self.load_config['time_embedding_mix'] = time_embedding_mix - log.debug(f'Control {what} model loading: id="{model_id}" path="{model_path}" {self.load_config}') - if model_path.endswith('.safetensors'): - self.model = ControlNetXSModel.from_single_file(model_path, **self.load_config) - else: - self.model = ControlNetXSModel.from_pretrained(model_path, **self.load_config) - if self.device is not None: - self.model.to(self.device) - if self.dtype is not None: - self.model.to(self.dtype) - t1 = time.time() - self.model_id = model_id - log.debug(f'Control {what} model loaded: id="{model_id}" path="{model_path}" time={t1-t0:.2f}') - return f'{what} loaded model: {model_id}' - except Exception as e: - log.error(f'Control {what} model load failed: id="{model_id}" error={e}') - errors.display(e, f'Control {what} load') - return f'{what} failed to load model: {model_id}' + with load_lock: + try: + t0 = time.time() + model_id = model_id or self.model_id + if model_id is None or model_id == 'None': + self.reset() + return + if model_id not in all_models: + log.error(f'Control {what} unknown model: id="{model_id}" available={list(all_models)}') + return + model_path = all_models[model_id] + if model_path == '': + return + if model_path is None: + log.error(f'Control {what} model load failed: id="{model_id}" error=unknown model id') + return + if model_id == self.model_id and not force: + # log.debug(f'Control {what} model: id="{model_id}" path="{model_path}" already loaded') + return + self.load_config['time_embedding_mix'] = time_embedding_mix + log.debug(f'Control {what} model loading: id="{model_id}" path="{model_path}" {self.load_config}') + if model_path.endswith('.safetensors'): + self.model = ControlNetXSModel.from_single_file(model_path, **self.load_config) + else: + self.model = ControlNetXSModel.from_pretrained(model_path, **self.load_config) + if self.device is not None: + self.model.to(self.device) + if self.dtype is not None: + self.model.to(self.dtype) + t1 = time.time() + self.model_id = model_id + log.debug(f'Control {what} model loaded: id="{model_id}" path="{model_path}" time={t1-t0:.2f}') + return f'{what} loaded model: {model_id}' + except Exception as e: + log.error(f'Control {what} model load failed: id="{model_id}" error={e}') + errors.display(e, f'Control {what} load') + return f'{what} failed to load model: {model_id}' class ControlNetXSPipeline(): diff --git a/modules/deepbooru.py b/modules/deepbooru.py index 719bd2737..f961a4351 100644 --- a/modules/deepbooru.py +++ b/modules/deepbooru.py @@ -1,11 +1,13 @@ import os import re +import threading import torch import numpy as np from PIL import Image from modules import modelloader, paths, deepbooru_model, devices, images, shared re_special = re.compile(r'([\\()])') +load_lock = threading.Lock() class DeepDanbooru: @@ -13,22 +15,23 @@ def __init__(self): self.model = None def load(self): - if self.model is not None: - return - model_path = os.path.join(paths.models_path, "DeepDanbooru") - shared.log.debug(f'Load interrogate model: type=DeepDanbooru folder="{model_path}"') - files = modelloader.load_models( - model_path=model_path, - model_url='https://github.com/AUTOMATIC1111/TorchDeepDanbooru/releases/download/v1/model-resnet_custom_v3.pt', - ext_filter=[".pt"], - download_name='model-resnet_custom_v3.pt', - ) + with load_lock: + if self.model is not None: + return + model_path = os.path.join(paths.models_path, "DeepDanbooru") + shared.log.debug(f'Load interrogate model: type=DeepDanbooru folder="{model_path}"') + files = modelloader.load_models( + model_path=model_path, + model_url='https://github.com/AUTOMATIC1111/TorchDeepDanbooru/releases/download/v1/model-resnet_custom_v3.pt', + ext_filter=[".pt"], + download_name='model-resnet_custom_v3.pt', + ) - self.model = deepbooru_model.DeepDanbooruModel() - self.model.load_state_dict(torch.load(files[0], map_location="cpu")) + self.model = deepbooru_model.DeepDanbooruModel() + self.model.load_state_dict(torch.load(files[0], map_location="cpu")) - self.model.eval() - self.model.to(devices.cpu, devices.dtype) + self.model.eval() + self.model.to(devices.cpu, devices.dtype) def start(self): self.load() diff --git a/modules/interrogate.py b/modules/interrogate.py index 6cdaae0c1..1cf9aee95 100644 --- a/modules/interrogate.py +++ b/modules/interrogate.py @@ -3,6 +3,7 @@ import time from collections import namedtuple from pathlib import Path +import threading import re import torch import torch.hub # pylint: disable=ungrouped-imports @@ -34,6 +35,7 @@ clip_model_name = 'ViT-L/14' Category = namedtuple("Category", ["name", "topn", "items"]) re_topn = re.compile(r"\.top(\d+)\.") +load_lock = threading.Lock() def category_types(): @@ -97,34 +99,35 @@ def checkpoint_wrapper(self): sys.modules["fairscale.nn.checkpoint.checkpoint_activations"] = FakeFairscale def load_blip_model(self): - self.create_fake_fairscale() - from repositories.blip import models # pylint: disable=unused-import - from repositories.blip.models import blip - import modules.modelloader as modelloader - model_path = os.path.join(paths.models_path, "BLIP") - download_name='model_base_caption_capfilt_large.pth' - shared.log.debug(f'Model interrogate load: type=BLiP model={download_name} path={model_path}') - files = modelloader.load_models( - model_path=model_path, - model_url='https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model_base_caption_capfilt_large.pth', - ext_filter=[".pth"], - download_name=download_name, - ) - blip_model = blip.blip_decoder(pretrained=files[0], image_size=blip_image_eval_size, vit='base', med_config=os.path.join(paths.paths["BLIP"], "configs", "med_config.json")) # pylint: disable=c-extension-no-member - blip_model.eval() - - return blip_model + with load_lock: + self.create_fake_fairscale() + from repositories.blip import models # pylint: disable=unused-import + from repositories.blip.models import blip + import modules.modelloader as modelloader + model_path = os.path.join(paths.models_path, "BLIP") + download_name='model_base_caption_capfilt_large.pth' + shared.log.debug(f'Model interrogate load: type=BLiP model={download_name} path={model_path}') + files = modelloader.load_models( + model_path=model_path, + model_url='https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model_base_caption_capfilt_large.pth', + ext_filter=[".pth"], + download_name=download_name, + ) + blip_model = blip.blip_decoder(pretrained=files[0], image_size=blip_image_eval_size, vit='base', med_config=os.path.join(paths.paths["BLIP"], "configs", "med_config.json")) # pylint: disable=c-extension-no-member + blip_model.eval() + return blip_model def load_clip_model(self): - shared.log.debug(f'Model interrogate load: type=CLiP model={clip_model_name} path={shared.opts.clip_models_path}') - import clip - if self.running_on_cpu: - model, preprocess = clip.load(clip_model_name, device="cpu", download_root=shared.opts.clip_models_path) - else: - model, preprocess = clip.load(clip_model_name, download_root=shared.opts.clip_models_path) - model.eval() - model = model.to(devices.device) - return model, preprocess + with load_lock: + shared.log.debug(f'Model interrogate load: type=CLiP model={clip_model_name} path={shared.opts.clip_models_path}') + import clip + if self.running_on_cpu: + model, preprocess = clip.load(clip_model_name, device="cpu", download_root=shared.opts.clip_models_path) + else: + model, preprocess = clip.load(clip_model_name, download_root=shared.opts.clip_models_path) + model.eval() + model = model.to(devices.device) + return model, preprocess def load(self): if self.blip_model is None: diff --git a/modules/paths.py b/modules/paths.py index 9a73a9f91..134bcd2ab 100644 --- a/modules/paths.py +++ b/modules/paths.py @@ -2,18 +2,20 @@ import os import sys import json +import shlex import argparse from modules.errors import log # parse args, parse again after we have the data-dir and early-read the config file +argv = shlex.split(" ".join(sys.argv[1:])) if "USED_VSCODE_COMMAND_PICKARGS" in os.environ else sys.argv[1:] parser = argparse.ArgumentParser(add_help=False) parser.add_argument("--ckpt", type=str, default=os.environ.get("SD_MODEL", None), help="Path to model checkpoint to load immediately, default: %(default)s") parser.add_argument("--data-dir", type=str, default=os.environ.get("SD_DATADIR", ''), help="Base path where all user data is stored, default: %(default)s") parser.add_argument("--models-dir", type=str, default=os.environ.get("SD_MODELSDIR", None), help="Base path where all models are stored, default: %(default)s",) -cli = parser.parse_known_args()[0] -parser.add_argument("--config", type=str, default=os.environ.get("SD_CONFIG", os.path.join(cli.data_dir, 'config.json')), help="Use specific server configuration file, default: %(default)s") -cli = parser.parse_known_args()[0] +cli = parser.parse_known_args(argv)[0] +parser.add_argument("--config", type=str, default=os.environ.get("SD_CONFIG", os.path.join(cli.data_dir, 'config.json')), help="Use specific server configuration file, default: %(default)s") # twice because we want data_dir +cli = parser.parse_known_args(argv)[0] config_path = cli.config if os.path.isabs(cli.config) else os.path.join(cli.data_dir, cli.config) try: with open(config_path, 'r', encoding='utf8') as f: diff --git a/modules/postprocess/yolo.py b/modules/postprocess/yolo.py index 57020b60f..85a50cc2a 100644 --- a/modules/postprocess/yolo.py +++ b/modules/postprocess/yolo.py @@ -1,5 +1,6 @@ from typing import TYPE_CHECKING import os +import threading from copy import copy import numpy as np import gradio as gr @@ -16,6 +17,7 @@ 'https://huggingface.co/vladmandic/yolo-detailers/resolve/main/eyes-v1.pt', 'https://huggingface.co/vladmandic/yolo-detailers/resolve/main/eyes-full-v1.pt', ] +load_lock = threading.Lock() class YoloResult: @@ -151,28 +153,29 @@ def predict( return result def load(self, model_name: str = None): - from modules import modelloader - model = None - self.dependencies() - if model_name is None: - model_name = list(self.list)[0] - if model_name in self.models: - return model_name, self.models[model_name] - else: - model_url = self.list.get(model_name) - file_name = os.path.basename(model_url) - model_file = None - try: - model_file = modelloader.load_file_from_url(url=model_url, model_dir=shared.opts.yolo_dir, file_name=file_name) - if model_file is not None: - import ultralytics - model = ultralytics.YOLO(model_file) - classes = list(model.names.values()) - shared.log.info(f'Load: type=Detailer name="{model_name}" model="{model_file}" ultralytics={ultralytics.__version__} classes={classes}') - self.models[model_name] = model - return model_name, model - except Exception as e: - shared.log.error(f'Load: type=Detailer name="{model_name}" error="{e}"') + with load_lock: + from modules import modelloader + model = None + self.dependencies() + if model_name is None: + model_name = list(self.list)[0] + if model_name in self.models: + return model_name, self.models[model_name] + else: + model_url = self.list.get(model_name) + file_name = os.path.basename(model_url) + model_file = None + try: + model_file = modelloader.load_file_from_url(url=model_url, model_dir=shared.opts.yolo_dir, file_name=file_name) + if model_file is not None: + import ultralytics + model = ultralytics.YOLO(model_file) + classes = list(model.names.values()) + shared.log.info(f'Load: type=Detailer name="{model_name}" model="{model_file}" ultralytics={ultralytics.__version__} classes={classes}') + self.models[model_name] = model + return model_name, model + except Exception as e: + shared.log.error(f'Load: type=Detailer name="{model_name}" error="{e}"') return None, None def restore(self, np_image, p: processing.StableDiffusionProcessing = None): diff --git a/modules/sd_checkpoint.py b/modules/sd_checkpoint.py index 6ab396329..ec41a130f 100644 --- a/modules/sd_checkpoint.py +++ b/modules/sd_checkpoint.py @@ -210,6 +210,8 @@ def get_closet_checkpoint_match(s: str): if shared.opts.sd_checkpoint_autodownload and s.count('/') == 1: modelloader.hf_login() found = modelloader.find_diffuser(s, full=True) + if found is None: + return None found = [f for f in found if f == s] shared.log.info(f'HF search: model="{s}" results={found}') if found is not None and len(found) == 1: @@ -262,7 +264,7 @@ def select_checkpoint(op='model'): shared.log.info(f'Load {op}: select="{checkpoint_info.title if checkpoint_info is not None else None}"') return checkpoint_info if len(checkpoints_list) == 0: - shared.log.warning("Cannot generate without a checkpoint") + shared.log.error("No models found") shared.log.info("Set system paths to use existing folders") shared.log.info(" or use --models-dir to specify base folder with all models") shared.log.info(" or use --ckpt-dir to specify folder with sd models") diff --git a/package.json b/package.json index 64115587d..95f8899bf 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ }, "scripts": { "venv": "source venv/bin/activate", - "start": "python launch.py --debug --experimental", + "start": "python launch.py --debug", "ruff": "ruff check", "eslint": "eslint javascript/ extensions-builtin/sdnext-modernui/javascript/", "pylint": "pylint *.py modules/ extensions-builtin/", diff --git a/scripts/prompt_enhance.py b/scripts/prompt_enhance.py index a02aea608..0ad17bba4 100644 --- a/scripts/prompt_enhance.py +++ b/scripts/prompt_enhance.py @@ -2,6 +2,7 @@ import time import random +import threading from transformers import AutoTokenizer, AutoModelForSeq2SeqLM import gradio as gr from modules import shared, scripts, devices, processing @@ -9,6 +10,7 @@ repo_id = "gokaygokay/Flux-Prompt-Enhance" num_return_sequences = 5 +load_lock = threading.Lock() class Script(scripts.Script): @@ -31,11 +33,12 @@ def show(self, is_img2img): return shared.native def load(self): - if self.tokenizer is None: - self.tokenizer = AutoTokenizer.from_pretrained('gokaygokay/Flux-Prompt-Enhance', cache_dir=shared.opts.hfcache_dir) - if self.model is None: - shared.log.info(f'Prompt enhance: model="{repo_id}"') - self.model = AutoModelForSeq2SeqLM.from_pretrained('gokaygokay/Flux-Prompt-Enhance', cache_dir=shared.opts.hfcache_dir).to(device=devices.cpu, dtype=devices.dtype) + with load_lock: + if self.tokenizer is None: + self.tokenizer = AutoTokenizer.from_pretrained('gokaygokay/Flux-Prompt-Enhance', cache_dir=shared.opts.hfcache_dir) + if self.model is None: + shared.log.info(f'Prompt enhance: model="{repo_id}"') + self.model = AutoModelForSeq2SeqLM.from_pretrained('gokaygokay/Flux-Prompt-Enhance', cache_dir=shared.opts.hfcache_dir).to(device=devices.cpu, dtype=devices.dtype) def enhance(self, prompt, auto_apply: bool = False, temperature: float = 0.7, repetition_penalty: float = 1.2, max_length: int = 128): self.load() From ba0b715af08951a1af74bfcbcc3c9420f63c41e6 Mon Sep 17 00:00:00 2001 From: Vladimir Mandic Date: Fri, 10 Jan 2025 13:23:56 -0500 Subject: [PATCH 246/249] add vscode launch config Signed-off-by: Vladimir Mandic --- .vscode/launch.json | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .vscode/launch.json diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 000000000..5fe45a869 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,23 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "SD.Next VSCode Debugger", + "type": "debugpy", + "request": "launch", + "program": "launch.py", + "cwd": "${workspaceFolder}", + "console": "integratedTerminal", + "env": { "USED_VSCODE_COMMAND_PICKARGS": "1" }, + "args": [ + "--uv", + "--quick", + "--debug", + "--docs", + "--api-log", + "--log", "vscode.log", + "${command:pickArgs}", + ] + } + ] +} From 49a7f30f9c3114d5ad78dfb621e2012bc263fdbf Mon Sep 17 00:00:00 2001 From: Seunghoon Lee Date: Sat, 11 Jan 2025 23:36:51 +0900 Subject: [PATCH 247/249] check hipblaslt availability in windows --- installer.py | 16 +++++---- modules/rocm.py | 18 ++++------ modules/zluda_hijacks.py | 2 +- modules/zluda_installer.py | 67 ++++++++++++++++++++------------------ 4 files changed, 53 insertions(+), 50 deletions(-) diff --git a/installer.py b/installer.py index 7b351c799..91496ac49 100644 --- a/installer.py +++ b/installer.py @@ -569,6 +569,7 @@ def install_rocm_zluda(): msg += f', using agent {device.name}' log.info(msg) torch_command = '' + if sys.platform == "win32": # TODO install: enable ROCm for windows when available @@ -584,17 +585,20 @@ def install_rocm_zluda(): try: if args.reinstall: zluda_installer.uninstall() - zluda_path = zluda_installer.get_path() - zluda_installer.install(zluda_path) - zluda_installer.make_copy(zluda_path) + zluda_installer.install() except Exception as e: error = e log.warning(f'Failed to install ZLUDA: {e}') + if error is None: try: - zluda_installer.load(zluda_path) + if device is not None and zluda_installer.get_blaslt_enabled(): + log.debug(f'ROCm hipBLASLt: arch={device.name} available={device.blaslt_supported}') + zluda_installer.set_blaslt_enabled(device.blaslt_supported) + zluda_installer.make_copy() + zluda_installer.load() torch_command = os.environ.get('TORCH_COMMAND', f'torch=={zluda_installer.get_default_torch_version(device)} torchvision --index-url https://download.pytorch.org/whl/cu118') - log.info(f'Using ZLUDA in {zluda_path}') + log.info(f'Using ZLUDA in {zluda_installer.path}') except Exception as e: error = e log.warning(f'Failed to load ZLUDA: {e}') @@ -631,7 +635,7 @@ def install_rocm_zluda(): #elif not args.experimental: # uninstall('flash-attn') - if device is not None and rocm.version != "6.2" and rocm.version == rocm.version_torch and rocm.get_blaslt_enabled(): + if device is not None and rocm.version != "6.2" and rocm.get_blaslt_enabled(): log.debug(f'ROCm hipBLASLt: arch={device.name} available={device.blaslt_supported}') rocm.set_blaslt_enabled(device.blaslt_supported) diff --git a/modules/rocm.py b/modules/rocm.py index ef76a1cfa..704ae19c8 100644 --- a/modules/rocm.py +++ b/modules/rocm.py @@ -8,10 +8,6 @@ from enum import Enum -HIPBLASLT_TENSILE_LIBPATH = os.environ.get("HIPBLASLT_TENSILE_LIBPATH", None if sys.platform == "win32" # not available - else "/opt/rocm/lib/hipblaslt/library") - - def resolve_link(path_: str) -> str: if not os.path.islink(path_): return path_ @@ -55,8 +51,7 @@ class Agent: gfx_version: int arch: MicroArchitecture is_apu: bool - if sys.platform != "win32": - blaslt_supported: bool + blaslt_supported: bool @staticmethod def parse_gfx_version(name: str) -> int: @@ -83,8 +78,7 @@ def __init__(self, name: str): else: self.arch = MicroArchitecture.GCN self.is_apu = (self.gfx_version & 0xFFF0 == 0x1150) or self.gfx_version in (0x801, 0x902, 0x90c, 0x1013, 0x1033, 0x1035, 0x1036, 0x1103,) - if sys.platform != "win32": - self.blaslt_supported = os.path.exists(os.path.join(HIPBLASLT_TENSILE_LIBPATH, f"extop_{name}.co")) + self.blaslt_supported = os.path.exists(os.path.join(blaslt_tensile_libpath, f"Kernels.so-000-{name}.hsaco" if sys.platform == "win32" else f"extop_{name}.co")) def get_gfx_version(self) -> Union[str, None]: if self.gfx_version >= 0x1200: @@ -163,6 +157,7 @@ def get_agents() -> List[Agent]: return [Agent(x.split(' ')[-1].strip()) for x in spawn("hipinfo", cwd=os.path.join(path, 'bin')).split("\n") if x.startswith('gcnArchName:')] is_wsl: bool = False + version_torch = None else: def find() -> Union[str, None]: rocm_path = shutil.which("hipconfig") @@ -199,12 +194,12 @@ def load_hsa_runtime() -> None: def set_blaslt_enabled(enabled: bool) -> None: if enabled: load_library_global("/opt/rocm/lib/libhipblaslt.so") # Preload hipBLASLt. - os.environ["HIPBLASLT_TENSILE_LIBPATH"] = HIPBLASLT_TENSILE_LIBPATH + os.environ["HIPBLASLT_TENSILE_LIBPATH"] = blaslt_tensile_libpath else: os.environ["TORCH_BLAS_PREFER_HIPBLASLT"] = "0" def get_blaslt_enabled() -> bool: - return bool(int(os.environ.get("TORCH_BLAS_PREFER_HIPBLASLT", "1"))) + return version == version_torch and bool(int(os.environ.get("TORCH_BLAS_PREFER_HIPBLASLT", "1"))) def get_flash_attention_command(agent: Agent): if os.environ.get("FLASH_ATTENTION_USE_TRITON_ROCM", "FALSE") == "TRUE": @@ -215,10 +210,11 @@ def get_flash_attention_command(agent: Agent): return os.environ.get("FLASH_ATTENTION_PACKAGE", default) is_wsl: bool = os.environ.get('WSL_DISTRO_NAME', 'unknown' if spawn('wslpath -w /') else None) is not None + version_torch = get_version_torch() path = find() +blaslt_tensile_libpath = os.environ.get("HIPBLASLT_TENSILE_LIBPATH", os.path.join(path, "bin" if sys.platform == "win32" else "lib", "hipblaslt", "library")) is_installed = False version = None -version_torch = get_version_torch() if path is not None: is_installed = True version = get_version() diff --git a/modules/zluda_hijacks.py b/modules/zluda_hijacks.py index cd41bcb72..01ca4dec8 100644 --- a/modules/zluda_hijacks.py +++ b/modules/zluda_hijacks.py @@ -36,5 +36,5 @@ def do_hijack(): torch.fft.ifftn = fft_ifftn torch.fft.rfftn = fft_rfftn - if not zluda_installer.experimental_hipBLASLt_support: + if not zluda_installer.get_blaslt_enabled(): torch.jit.script = jit_script diff --git a/modules/zluda_installer.py b/modules/zluda_installer.py index a3cc6d7ab..6c74ee6b8 100644 --- a/modules/zluda_installer.py +++ b/modules/zluda_installer.py @@ -16,12 +16,10 @@ } HIPSDK_TARGETS = ['rocblas.dll', 'rocsolver.dll'] ZLUDA_TARGETS = ('nvcuda.dll', 'nvml.dll',) -experimental_hipBLASLt_support: bool = False -default_agent: Union[rocm.Agent, None] = None - -def get_path() -> str: - return os.path.abspath(os.environ.get('ZLUDA', '.zluda')) +path = os.path.abspath(os.environ.get('ZLUDA', '.zluda')) +default_agent: Union[rocm.Agent, None] = None +hipBLASLt_enabled = os.path.exists(os.path.join(rocm.path, "bin", "hipblaslt.dll")) and os.path.exists(rocm.blaslt_tensile_libpath) and os.path.exists(os.path.join(path, 'cublasLt.dll')) def set_default_agent(agent: rocm.Agent): @@ -29,9 +27,8 @@ def set_default_agent(agent: rocm.Agent): default_agent = agent -def install(zluda_path: os.PathLike) -> None: - if os.path.exists(zluda_path): - __initialize(zluda_path) +def install() -> None: + if os.path.exists(path): return platform = "windows" @@ -46,7 +43,6 @@ def install(zluda_path: os.PathLike) -> None: info.filename = os.path.basename(info.filename) archive.extract(info, '.zluda') os.remove('_zluda') - __initialize(zluda_path) def uninstall() -> None: @@ -54,27 +50,45 @@ def uninstall() -> None: shutil.rmtree('.zluda') -def make_copy(zluda_path: os.PathLike) -> None: - __initialize(zluda_path) +def set_blaslt_enabled(enabled: bool): + global hipBLASLt_enabled # pylint: disable=global-statement + hipBLASLt_enabled = enabled + + +def get_blaslt_enabled() -> bool: + return hipBLASLt_enabled + + +def link_or_copy(src: os.PathLike, dst: os.PathLike): + try: + os.link(src, dst) + except Exception: + shutil.copyfile(src, dst) + +def make_copy() -> None: for k, v in DLL_MAPPING.items(): - if not os.path.exists(os.path.join(zluda_path, v)): - try: - os.link(os.path.join(zluda_path, k), os.path.join(zluda_path, v)) - except Exception: - shutil.copyfile(os.path.join(zluda_path, k), os.path.join(zluda_path, v)) + if not os.path.exists(os.path.join(path, v)): + link_or_copy(os.path.join(path, k), os.path.join(path, v)) + + if hipBLASLt_enabled and not os.path.exists(os.path.join(path, 'cublasLt64_11.dll')): + link_or_copy(os.path.join(path, 'cublasLt.dll'), os.path.join(path, 'cublasLt64_11.dll')) -def load(zluda_path: os.PathLike) -> None: +def load() -> None: os.environ["ZLUDA_COMGR_LOG_LEVEL"] = "1" os.environ["ZLUDA_NVRTC_LIB"] = os.path.join([v for v in site.getsitepackages() if v.endswith("site-packages")][0], "torch", "lib", "nvrtc64_112_0.dll") for v in HIPSDK_TARGETS: ctypes.windll.LoadLibrary(os.path.join(rocm.path, 'bin', v)) for v in ZLUDA_TARGETS: - ctypes.windll.LoadLibrary(os.path.join(zluda_path, v)) + ctypes.windll.LoadLibrary(os.path.join(path, v)) for v in DLL_MAPPING.values(): - ctypes.windll.LoadLibrary(os.path.join(zluda_path, v)) + ctypes.windll.LoadLibrary(os.path.join(path, v)) + + if hipBLASLt_enabled: + ctypes.windll.LoadLibrary(os.path.join(rocm.path, 'bin', 'hipblaslt.dll')) + ctypes.windll.LoadLibrary(os.path.join(path, 'cublasLt64_11.dll')) def conceal(): import torch # pylint: disable=unused-import @@ -94,18 +108,7 @@ def _join_rocm_home(*paths) -> str: def get_default_torch_version(agent: Optional[rocm.Agent]) -> str: if agent is not None: if agent.arch in (rocm.MicroArchitecture.RDNA, rocm.MicroArchitecture.CDNA,): - return "2.4.1" if experimental_hipBLASLt_support else "2.3.1" + return "2.4.1" if hipBLASLt_enabled else "2.3.1" elif agent.arch == rocm.MicroArchitecture.GCN: return "2.2.1" - return "2.4.1" if experimental_hipBLASLt_support else "2.3.1" - - -def __initialize(zluda_path: os.PathLike): - global experimental_hipBLASLt_support # pylint: disable=global-statement - experimental_hipBLASLt_support = os.path.exists(os.path.join(zluda_path, 'cublasLt.dll')) - - if experimental_hipBLASLt_support: - HIPSDK_TARGETS.append('hipblaslt.dll') - DLL_MAPPING['cublasLt.dll'] = 'cublasLt64_11.dll' - else: - HIPSDK_TARGETS.append(f'hiprtc{"".join([v.zfill(2) for v in rocm.version.split(".")])}.dll') + return "2.4.1" if hipBLASLt_enabled else "2.3.1" From 29143930a1cf20bcfcdad677dd973cdcf12a6649 Mon Sep 17 00:00:00 2001 From: Seunghoon Lee Date: Sat, 11 Jan 2025 23:41:17 +0900 Subject: [PATCH 248/249] Fix bug. --- modules/rocm.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/rocm.py b/modules/rocm.py index 704ae19c8..05fb2260e 100644 --- a/modules/rocm.py +++ b/modules/rocm.py @@ -212,9 +212,10 @@ def get_flash_attention_command(agent: Agent): is_wsl: bool = os.environ.get('WSL_DISTRO_NAME', 'unknown' if spawn('wslpath -w /') else None) is not None version_torch = get_version_torch() path = find() -blaslt_tensile_libpath = os.environ.get("HIPBLASLT_TENSILE_LIBPATH", os.path.join(path, "bin" if sys.platform == "win32" else "lib", "hipblaslt", "library")) +blaslt_tensile_libpath = None is_installed = False version = None if path is not None: + blaslt_tensile_libpath = os.environ.get("HIPBLASLT_TENSILE_LIBPATH", os.path.join(path, "bin" if sys.platform == "win32" else "lib", "hipblaslt", "library")) is_installed = True version = get_version() From 9ee392173c0de7f79c16ec87a47221294773e188 Mon Sep 17 00:00:00 2001 From: Disty0 Date: Sun, 12 Jan 2025 00:29:26 +0300 Subject: [PATCH 249/249] Move jxl plugin to optional pkgs --- installer.py | 1 + modules/loader.py | 5 ++++- requirements.txt | 1 - 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/installer.py b/installer.py index 91496ac49..9ad03a862 100644 --- a/installer.py +++ b/installer.py @@ -1050,6 +1050,7 @@ def install_optional(): install('basicsr') install('gfpgan') install('clean-fid') + install('pillow-jxl-plugin==1.3.1', ignore=True) install('optimum-quanto=0.2.6', ignore=True) install('bitsandbytes==0.45.0', ignore=True) install('pynvml', ignore=True) diff --git a/modules/loader.py b/modules/loader.py index 8980fd037..63c52d18c 100644 --- a/modules/loader.py +++ b/modules/loader.py @@ -72,7 +72,10 @@ logging.getLogger("diffusers.loaders.single_file").setLevel(logging.ERROR) timer.startup.record("diffusers") -import pillow_jxl # pylint: disable=W0611,C0411 +try: + import pillow_jxl # pylint: disable=W0611,C0411 +except: + pass from PIL import Image # pylint: disable=W0611,C0411 timer.startup.record("pillow") diff --git a/requirements.txt b/requirements.txt index 268082699..c8ed07a40 100644 --- a/requirements.txt +++ b/requirements.txt @@ -55,7 +55,6 @@ tokenizers==0.21.0 transformers==4.47.1 urllib3==1.26.19 Pillow==10.4.0 -pillow-jxl-plugin==1.3.0 timm==0.9.16 pydantic==1.10.15 pyparsing==3.1.4