On this page https://github.com/madebyollin/taesdv , it mentions that it supports Streamdiffusion
What I have attempted so far: Download taesdv.py and taesdv.pth into the same directory as my Python script, then do the following:
from taesdv import TAESDV
import numpy as np
from streamdiffusion import StreamDiffusion
import torch
from diffusers import StableDiffusionPipeline
from streamdiffusion.acceleration.tensorrt import accelerate_with_tensorrt
from PIL import Image
INFERENCESTEPS = 17
indexlist=[12]
BATCH_SIZE = len(indexlist)+1
WIDTH, HEIGHT = 512,512
TORCH_DEVICE = 'cuda'
TORCH_DTYPE = torch.float16
PROMPT = "santa claus, thick frame glasses"
model_loc = "models/sd-turbo"
taesdv = TAESDV("taesdv.pth").half().cuda()
pipe = StableDiffusionPipeline.from_pretrained(model_loc).to(
device=torch.device(TORCH_DEVICE),
dtype=TORCH_DTYPE,
)
stream = StreamDiffusion(
pipe,
t_index_list=indexlist,
torch_dtype=TORCH_DTYPE,
frame_buffer_size=1,
width=WIDTH,
height=HEIGHT,
do_add_noise=True,
cfg_type='self'
)
stream = accelerate_with_tensorrt(
stream, "engines", max_batch_size=BATCH_SIZE
)
stream.vae = taesdv
stream.prepare(prompt = PROMPT,
num_inference_steps=INFERENCESTEPS,
guidance_scale=1)
testarray = np.zeros((HEIGHT, WIDTH,3),np.uint8)
testimage = Image.fromarray((testarray))
x_output = stream(testimage)
This results in the following:
File "C:\Users\fdasfasdfasd\Documents\test.py", line 47, in
x_output = stream(testimage)
File "C:\Users\fdasfasdfasd\AppData\Local\Programs\Python\Python310\lib\site-packages\torch\utils_contextlib.py", line 115, in decorate_context
return func(*args, **kwargs)
File "C:\Users\fdasfasdfasd\AppData\Local\Programs\Python\Python310\lib\site-packages\streamdiffusion\pipeline.py", line 455, in call
x_t_latent = self.encode_image(x)
File "C:\Users\fdasfasdfasd\AppData\Local\Programs\Python\Python310\lib\site-packages\streamdiffusion\pipeline.py", line 376, in encode_image
dtype=self.vae.dtype,
File "C:\Users\fdasfasdfasd\AppData\Local\Programs\Python\Python310\lib\site-packages\torch\nn\modules\module.py", line 1695, in getattr
raise AttributeError(f"'{type(self).name}' object has no attribute '{name}'")
AttributeError: 'TAESDV' object has no attribute 'dtype'. Did you mean: 'type'?
On this page https://github.com/madebyollin/taesdv , it mentions that it supports Streamdiffusion
What I have attempted so far: Download taesdv.py and taesdv.pth into the same directory as my Python script, then do the following:
This results in the following:
File "C:\Users\fdasfasdfasd\Documents\test.py", line 47, in
x_output = stream(testimage)
File "C:\Users\fdasfasdfasd\AppData\Local\Programs\Python\Python310\lib\site-packages\torch\utils_contextlib.py", line 115, in decorate_context
return func(*args, **kwargs)
File "C:\Users\fdasfasdfasd\AppData\Local\Programs\Python\Python310\lib\site-packages\streamdiffusion\pipeline.py", line 455, in call
x_t_latent = self.encode_image(x)
File "C:\Users\fdasfasdfasd\AppData\Local\Programs\Python\Python310\lib\site-packages\streamdiffusion\pipeline.py", line 376, in encode_image
dtype=self.vae.dtype,
File "C:\Users\fdasfasdfasd\AppData\Local\Programs\Python\Python310\lib\site-packages\torch\nn\modules\module.py", line 1695, in getattr
raise AttributeError(f"'{type(self).name}' object has no attribute '{name}'")
AttributeError: 'TAESDV' object has no attribute 'dtype'. Did you mean: 'type'?