mirror of
https://github.com/leejet/stable-diffusion.cpp.git
synced 2026-11-11 03:24:30 +00:00
refactor: reorganize the file structure (#1266)
This commit is contained in:
leejet authored and GitHub committed
1 parent
3296545090
commit
28ef93c0e1
51 files changed
+327
-326
No files matched your search
+6
-5
@@ -87,9 +87,9 @@ endif()
|
|||||||
set(SD_LIB stable-diffusion)
|
set(SD_LIB stable-diffusion)
|
||||||
|
|
||||||
file(GLOB SD_LIB_SOURCES
|
file(GLOB SD_LIB_SOURCES
|
||||||
"*.h"
|
"src/*.h"
|
||||||
"*.cpp"
|
"src/*.cpp"
|
||||||
"*.hpp"
|
"src/*.hpp"
|
||||||
)
|
)
|
||||||
|
|
||||||
find_program(GIT_EXE NAMES git git.exe NO_CMAKE_FIND_ROOT_PATH)
|
find_program(GIT_EXE NAMES git git.exe NO_CMAKE_FIND_ROOT_PATH)
|
||||||
@@ -119,7 +119,7 @@ endif()
|
|||||||
message(STATUS "stable-diffusion.cpp commit ${SDCPP_BUILD_COMMIT}")
|
message(STATUS "stable-diffusion.cpp commit ${SDCPP_BUILD_COMMIT}")
|
||||||
|
|
||||||
set_property(
|
set_property(
|
||||||
SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/version.cpp
|
SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/src/version.cpp
|
||||||
APPEND PROPERTY COMPILE_DEFINITIONS
|
APPEND PROPERTY COMPILE_DEFINITIONS
|
||||||
SDCPP_BUILD_COMMIT=${SDCPP_BUILD_COMMIT} SDCPP_BUILD_VERSION=${SDCPP_BUILD_VERSION}
|
SDCPP_BUILD_COMMIT=${SDCPP_BUILD_COMMIT} SDCPP_BUILD_VERSION=${SDCPP_BUILD_VERSION}
|
||||||
)
|
)
|
||||||
@@ -182,6 +182,7 @@ endif()
|
|||||||
add_subdirectory(thirdparty)
|
add_subdirectory(thirdparty)
|
||||||
|
|
||||||
target_link_libraries(${SD_LIB} PUBLIC ggml zip)
|
target_link_libraries(${SD_LIB} PUBLIC ggml zip)
|
||||||
|
target_include_directories(${SD_LIB} PUBLIC . include)
|
||||||
target_include_directories(${SD_LIB} PUBLIC . thirdparty)
|
target_include_directories(${SD_LIB} PUBLIC . thirdparty)
|
||||||
target_compile_features(${SD_LIB} PUBLIC c_std_11 cxx_std_17)
|
target_compile_features(${SD_LIB} PUBLIC c_std_11 cxx_std_17)
|
||||||
|
|
||||||
@@ -190,7 +191,7 @@ if (SD_BUILD_EXAMPLES)
|
|||||||
add_subdirectory(examples)
|
add_subdirectory(examples)
|
||||||
endif()
|
endif()
|
||||||
|
|
||||||
set(SD_PUBLIC_HEADERS stable-diffusion.h)
|
set(SD_PUBLIC_HEADERS include/stable-diffusion.h)
|
||||||
set_target_properties(${SD_LIB} PROPERTIES PUBLIC_HEADER "${SD_PUBLIC_HEADERS}")
|
set_target_properties(${SD_LIB} PROPERTIES PUBLIC_HEADER "${SD_PUBLIC_HEADERS}")
|
||||||
|
|
||||||
install(TARGETS ${SD_LIB} LIBRARY PUBLIC_HEADER)
|
install(TARGETS ${SD_LIB} LIBRARY PUBLIC_HEADER)
|
||||||
File renamed without changes.
@@ -1,88 +1,88 @@
|
|||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
import torch
|
import torch
|
||||||
from diffusers.utils import load_image
|
from diffusers.utils import load_image
|
||||||
# pip install insightface==0.7.3
|
# pip install insightface==0.7.3
|
||||||
from insightface.app import FaceAnalysis
|
from insightface.app import FaceAnalysis
|
||||||
from insightface.data import get_image as ins_get_image
|
from insightface.data import get_image as ins_get_image
|
||||||
from safetensors.torch import save_file
|
from safetensors.torch import save_file
|
||||||
|
|
||||||
###
|
###
|
||||||
# https://github.com/cubiq/ComfyUI_IPAdapter_plus/issues/165#issue-2055829543
|
# https://github.com/cubiq/ComfyUI_IPAdapter_plus/issues/165#issue-2055829543
|
||||||
###
|
###
|
||||||
class FaceAnalysis2(FaceAnalysis):
|
class FaceAnalysis2(FaceAnalysis):
|
||||||
# NOTE: allows setting det_size for each detection call.
|
# NOTE: allows setting det_size for each detection call.
|
||||||
# the model allows it but the wrapping code from insightface
|
# the model allows it but the wrapping code from insightface
|
||||||
# doesn't show it, and people end up loading duplicate models
|
# doesn't show it, and people end up loading duplicate models
|
||||||
# for different sizes where there is absolutely no need to
|
# for different sizes where there is absolutely no need to
|
||||||
def get(self, img, max_num=0, det_size=(640, 640)):
|
def get(self, img, max_num=0, det_size=(640, 640)):
|
||||||
if det_size is not None:
|
if det_size is not None:
|
||||||
self.det_model.input_size = det_size
|
self.det_model.input_size = det_size
|
||||||
|
|
||||||
return super().get(img, max_num)
|
return super().get(img, max_num)
|
||||||
|
|
||||||
def analyze_faces(face_analysis: FaceAnalysis, img_data: np.ndarray, det_size=(640, 640)):
|
def analyze_faces(face_analysis: FaceAnalysis, img_data: np.ndarray, det_size=(640, 640)):
|
||||||
# NOTE: try detect faces, if no faces detected, lower det_size until it does
|
# NOTE: try detect faces, if no faces detected, lower det_size until it does
|
||||||
detection_sizes = [None] + [(size, size) for size in range(640, 256, -64)] + [(256, 256)]
|
detection_sizes = [None] + [(size, size) for size in range(640, 256, -64)] + [(256, 256)]
|
||||||
|
|
||||||
for size in detection_sizes:
|
for size in detection_sizes:
|
||||||
faces = face_analysis.get(img_data, det_size=size)
|
faces = face_analysis.get(img_data, det_size=size)
|
||||||
if len(faces) > 0:
|
if len(faces) > 0:
|
||||||
return faces
|
return faces
|
||||||
|
|
||||||
return []
|
return []
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
#face_detector = FaceAnalysis2(providers=['CUDAExecutionProvider'], allowed_modules=['detection', 'recognition'])
|
#face_detector = FaceAnalysis2(providers=['CUDAExecutionProvider'], allowed_modules=['detection', 'recognition'])
|
||||||
face_detector = FaceAnalysis2(providers=['CPUExecutionProvider'], allowed_modules=['detection', 'recognition'])
|
face_detector = FaceAnalysis2(providers=['CPUExecutionProvider'], allowed_modules=['detection', 'recognition'])
|
||||||
face_detector.prepare(ctx_id=0, det_size=(640, 640))
|
face_detector.prepare(ctx_id=0, det_size=(640, 640))
|
||||||
#input_folder_name = './scarletthead_woman'
|
#input_folder_name = './scarletthead_woman'
|
||||||
input_folder_name = sys.argv[1]
|
input_folder_name = sys.argv[1]
|
||||||
image_basename_list = os.listdir(input_folder_name)
|
image_basename_list = os.listdir(input_folder_name)
|
||||||
image_path_list = sorted([os.path.join(input_folder_name, basename) for basename in image_basename_list])
|
image_path_list = sorted([os.path.join(input_folder_name, basename) for basename in image_basename_list])
|
||||||
|
|
||||||
input_id_images = []
|
input_id_images = []
|
||||||
for image_path in image_path_list:
|
for image_path in image_path_list:
|
||||||
input_id_images.append(load_image(image_path))
|
input_id_images.append(load_image(image_path))
|
||||||
|
|
||||||
id_embed_list = []
|
id_embed_list = []
|
||||||
|
|
||||||
for img in input_id_images:
|
for img in input_id_images:
|
||||||
img = np.array(img)
|
img = np.array(img)
|
||||||
img = img[:, :, ::-1]
|
img = img[:, :, ::-1]
|
||||||
faces = analyze_faces(face_detector, img)
|
faces = analyze_faces(face_detector, img)
|
||||||
if len(faces) > 0:
|
if len(faces) > 0:
|
||||||
id_embed_list.append(torch.from_numpy((faces[0]['embedding'])))
|
id_embed_list.append(torch.from_numpy((faces[0]['embedding'])))
|
||||||
|
|
||||||
if len(id_embed_list) == 0:
|
if len(id_embed_list) == 0:
|
||||||
raise ValueError(f"No face detected in input image pool")
|
raise ValueError(f"No face detected in input image pool")
|
||||||
|
|
||||||
id_embeds = torch.stack(id_embed_list)
|
id_embeds = torch.stack(id_embed_list)
|
||||||
|
|
||||||
# for r in id_embeds:
|
# for r in id_embeds:
|
||||||
# print(r)
|
# print(r)
|
||||||
# #torch.save(id_embeds, input_folder_name+'/id_embeds.pt');
|
# #torch.save(id_embeds, input_folder_name+'/id_embeds.pt');
|
||||||
# weights = dict()
|
# weights = dict()
|
||||||
# weights["id_embeds"] = id_embeds
|
# weights["id_embeds"] = id_embeds
|
||||||
# save_file(weights, input_folder_name+'/id_embeds.safetensors')
|
# save_file(weights, input_folder_name+'/id_embeds.safetensors')
|
||||||
|
|
||||||
binary_data = id_embeds.numpy().tobytes()
|
binary_data = id_embeds.numpy().tobytes()
|
||||||
two = 4
|
two = 4
|
||||||
zero = 0
|
zero = 0
|
||||||
one = 1
|
one = 1
|
||||||
tensor_name = "id_embeds"
|
tensor_name = "id_embeds"
|
||||||
# Write binary data to a file
|
# Write binary data to a file
|
||||||
with open(input_folder_name+'/id_embeds.bin', "wb") as f:
|
with open(input_folder_name+'/id_embeds.bin', "wb") as f:
|
||||||
f.write(two.to_bytes(4, byteorder='little'))
|
f.write(two.to_bytes(4, byteorder='little'))
|
||||||
f.write((len(tensor_name)).to_bytes(4, byteorder='little'))
|
f.write((len(tensor_name)).to_bytes(4, byteorder='little'))
|
||||||
f.write(zero.to_bytes(4, byteorder='little'))
|
f.write(zero.to_bytes(4, byteorder='little'))
|
||||||
f.write((id_embeds.shape[1]).to_bytes(4, byteorder='little'))
|
f.write((id_embeds.shape[1]).to_bytes(4, byteorder='little'))
|
||||||
f.write((id_embeds.shape[0]).to_bytes(4, byteorder='little'))
|
f.write((id_embeds.shape[0]).to_bytes(4, byteorder='little'))
|
||||||
f.write(one.to_bytes(4, byteorder='little'))
|
f.write(one.to_bytes(4, byteorder='little'))
|
||||||
f.write(one.to_bytes(4, byteorder='little'))
|
f.write(one.to_bytes(4, byteorder='little'))
|
||||||
f.write(tensor_name.encode('ascii'))
|
f.write(tensor_name.encode('ascii'))
|
||||||
f.write(binary_data)
|
f.write(binary_data)
|
||||||
|
|
||||||
|
|
||||||
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
@@ -1,234 +1,234 @@
|
|||||||
#include <cstddef>
|
#include <cstddef>
|
||||||
#include <cstdint>
|
#include <cstdint>
|
||||||
#include "ggml.h"
|
#include "ggml.h"
|
||||||
|
|
||||||
const float wan_21_latent_rgb_proj[16][3] = {
|
const float wan_21_latent_rgb_proj[16][3] = {
|
||||||
{0.015123f, -0.148418f, 0.479828f},
|
{0.015123f, -0.148418f, 0.479828f},
|
||||||
{0.003652f, -0.010680f, -0.037142f},
|
{0.003652f, -0.010680f, -0.037142f},
|
||||||
{0.212264f, 0.063033f, 0.016779f},
|
{0.212264f, 0.063033f, 0.016779f},
|
||||||
{0.232999f, 0.406476f, 0.220125f},
|
{0.232999f, 0.406476f, 0.220125f},
|
||||||
{-0.051864f, -0.082384f, -0.069396f},
|
{-0.051864f, -0.082384f, -0.069396f},
|
||||||
{0.085005f, -0.161492f, 0.010689f},
|
{0.085005f, -0.161492f, 0.010689f},
|
||||||
{-0.245369f, -0.506846f, -0.117010f},
|
{-0.245369f, -0.506846f, -0.117010f},
|
||||||
{-0.151145f, 0.017721f, 0.007207f},
|
{-0.151145f, 0.017721f, 0.007207f},
|
||||||
{-0.293239f, -0.207936f, -0.421135f},
|
{-0.293239f, -0.207936f, -0.421135f},
|
||||||
{-0.187721f, 0.050783f, 0.177649f},
|
{-0.187721f, 0.050783f, 0.177649f},
|
||||||
{-0.013067f, 0.265964f, 0.166578f},
|
{-0.013067f, 0.265964f, 0.166578f},
|
||||||
{0.028327f, 0.109329f, 0.108642f},
|
{0.028327f, 0.109329f, 0.108642f},
|
||||||
{-0.205343f, 0.043991f, 0.148914f},
|
{-0.205343f, 0.043991f, 0.148914f},
|
||||||
{0.014307f, -0.048647f, -0.007219f},
|
{0.014307f, -0.048647f, -0.007219f},
|
||||||
{0.217150f, 0.053074f, 0.319923f},
|
{0.217150f, 0.053074f, 0.319923f},
|
||||||
{0.155357f, 0.083156f, 0.064780f}};
|
{0.155357f, 0.083156f, 0.064780f}};
|
||||||
float wan_21_latent_rgb_bias[3] = {-0.270270f, -0.234976f, -0.456853f};
|
float wan_21_latent_rgb_bias[3] = {-0.270270f, -0.234976f, -0.456853f};
|
||||||
|
|
||||||
const float wan_22_latent_rgb_proj[48][3] = {
|
const float wan_22_latent_rgb_proj[48][3] = {
|
||||||
{0.017126f, -0.027230f, -0.019257f},
|
{0.017126f, -0.027230f, -0.019257f},
|
||||||
{-0.113739f, -0.028715f, -0.022885f},
|
{-0.113739f, -0.028715f, -0.022885f},
|
||||||
{-0.000106f, 0.021494f, 0.004629f},
|
{-0.000106f, 0.021494f, 0.004629f},
|
||||||
{-0.013273f, -0.107137f, -0.033638f},
|
{-0.013273f, -0.107137f, -0.033638f},
|
||||||
{-0.000381f, 0.000279f, 0.025877f},
|
{-0.000381f, 0.000279f, 0.025877f},
|
||||||
{-0.014216f, -0.003975f, 0.040528f},
|
{-0.014216f, -0.003975f, 0.040528f},
|
||||||
{0.001638f, -0.000748f, 0.011022f},
|
{0.001638f, -0.000748f, 0.011022f},
|
||||||
{0.029238f, -0.006697f, 0.035933f},
|
{0.029238f, -0.006697f, 0.035933f},
|
||||||
{0.021641f, -0.015874f, 0.040531f},
|
{0.021641f, -0.015874f, 0.040531f},
|
||||||
{-0.101984f, -0.070160f, -0.028855f},
|
{-0.101984f, -0.070160f, -0.028855f},
|
||||||
{0.033207f, -0.021068f, 0.002663f},
|
{0.033207f, -0.021068f, 0.002663f},
|
||||||
{-0.104711f, 0.121673f, 0.102981f},
|
{-0.104711f, 0.121673f, 0.102981f},
|
||||||
{0.082647f, -0.004991f, 0.057237f},
|
{0.082647f, -0.004991f, 0.057237f},
|
||||||
{-0.027375f, 0.031581f, 0.006868f},
|
{-0.027375f, 0.031581f, 0.006868f},
|
||||||
{-0.045434f, 0.029444f, 0.019287f},
|
{-0.045434f, 0.029444f, 0.019287f},
|
||||||
{-0.046572f, -0.012537f, 0.006675f},
|
{-0.046572f, -0.012537f, 0.006675f},
|
||||||
{0.074709f, 0.033690f, 0.025289f},
|
{0.074709f, 0.033690f, 0.025289f},
|
||||||
{-0.008251f, -0.002745f, -0.006999f},
|
{-0.008251f, -0.002745f, -0.006999f},
|
||||||
{0.012685f, -0.061856f, -0.048658f},
|
{0.012685f, -0.061856f, -0.048658f},
|
||||||
{0.042304f, -0.007039f, 0.000295f},
|
{0.042304f, -0.007039f, 0.000295f},
|
||||||
{-0.007644f, -0.060843f, -0.033142f},
|
{-0.007644f, -0.060843f, -0.033142f},
|
||||||
{0.159909f, 0.045628f, 0.367541f},
|
{0.159909f, 0.045628f, 0.367541f},
|
||||||
{0.095171f, 0.086438f, 0.010271f},
|
{0.095171f, 0.086438f, 0.010271f},
|
||||||
{0.006812f, 0.019643f, 0.029637f},
|
{0.006812f, 0.019643f, 0.029637f},
|
||||||
{0.003467f, -0.010705f, 0.014252f},
|
{0.003467f, -0.010705f, 0.014252f},
|
||||||
{-0.099681f, -0.066272f, -0.006243f},
|
{-0.099681f, -0.066272f, -0.006243f},
|
||||||
{0.047357f, 0.037040f, 0.000185f},
|
{0.047357f, 0.037040f, 0.000185f},
|
||||||
{-0.041797f, -0.089225f, -0.032257f},
|
{-0.041797f, -0.089225f, -0.032257f},
|
||||||
{0.008928f, 0.017028f, 0.018684f},
|
{0.008928f, 0.017028f, 0.018684f},
|
||||||
{-0.042255f, 0.016045f, 0.006849f},
|
{-0.042255f, 0.016045f, 0.006849f},
|
||||||
{0.011268f, 0.036462f, 0.037387f},
|
{0.011268f, 0.036462f, 0.037387f},
|
||||||
{0.011553f, -0.016375f, -0.048589f},
|
{0.011553f, -0.016375f, -0.048589f},
|
||||||
{0.046266f, -0.027189f, 0.056979f},
|
{0.046266f, -0.027189f, 0.056979f},
|
||||||
{0.009640f, -0.017576f, 0.030324f},
|
{0.009640f, -0.017576f, 0.030324f},
|
||||||
{-0.045794f, -0.036083f, -0.010616f},
|
{-0.045794f, -0.036083f, -0.010616f},
|
||||||
{0.022418f, 0.039783f, -0.032939f},
|
{0.022418f, 0.039783f, -0.032939f},
|
||||||
{-0.052714f, -0.015525f, 0.007438f},
|
{-0.052714f, -0.015525f, 0.007438f},
|
||||||
{0.193004f, 0.223541f, 0.264175f},
|
{0.193004f, 0.223541f, 0.264175f},
|
||||||
{-0.059406f, -0.008188f, 0.022867f},
|
{-0.059406f, -0.008188f, 0.022867f},
|
||||||
{-0.156742f, -0.263791f, -0.007385f},
|
{-0.156742f, -0.263791f, -0.007385f},
|
||||||
{-0.015717f, 0.016570f, 0.033969f},
|
{-0.015717f, 0.016570f, 0.033969f},
|
||||||
{0.037969f, 0.109835f, 0.200449f},
|
{0.037969f, 0.109835f, 0.200449f},
|
||||||
{-0.000782f, -0.009566f, -0.008058f},
|
{-0.000782f, -0.009566f, -0.008058f},
|
||||||
{0.010709f, 0.052960f, -0.044195f},
|
{0.010709f, 0.052960f, -0.044195f},
|
||||||
{0.017271f, 0.045839f, 0.034569f},
|
{0.017271f, 0.045839f, 0.034569f},
|
||||||
{0.009424f, 0.013088f, -0.001714f},
|
{0.009424f, 0.013088f, -0.001714f},
|
||||||
{-0.024805f, -0.059378f, -0.033756f},
|
{-0.024805f, -0.059378f, -0.033756f},
|
||||||
{-0.078293f, 0.029070f, 0.026129f}};
|
{-0.078293f, 0.029070f, 0.026129f}};
|
||||||
float wan_22_latent_rgb_bias[3] = {0.013160f, -0.096492f, -0.071323f};
|
float wan_22_latent_rgb_bias[3] = {0.013160f, -0.096492f, -0.071323f};
|
||||||
|
|
||||||
const float flux_latent_rgb_proj[16][3] = {
|
const float flux_latent_rgb_proj[16][3] = {
|
||||||
{-0.041168f, 0.019917f, 0.097253f},
|
{-0.041168f, 0.019917f, 0.097253f},
|
||||||
{0.028096f, 0.026730f, 0.129576f},
|
{0.028096f, 0.026730f, 0.129576f},
|
||||||
{0.065618f, -0.067950f, -0.014651f},
|
{0.065618f, -0.067950f, -0.014651f},
|
||||||
{-0.012998f, -0.014762f, 0.081251f},
|
{-0.012998f, -0.014762f, 0.081251f},
|
||||||
{0.078567f, 0.059296f, -0.024687f},
|
{0.078567f, 0.059296f, -0.024687f},
|
||||||
{-0.015987f, -0.003697f, 0.005012f},
|
{-0.015987f, -0.003697f, 0.005012f},
|
||||||
{0.033605f, 0.138999f, 0.068517f},
|
{0.033605f, 0.138999f, 0.068517f},
|
||||||
{-0.024450f, -0.063567f, -0.030101f},
|
{-0.024450f, -0.063567f, -0.030101f},
|
||||||
{-0.040194f, -0.016710f, 0.127185f},
|
{-0.040194f, -0.016710f, 0.127185f},
|
||||||
{0.112681f, 0.088764f, -0.041940f},
|
{0.112681f, 0.088764f, -0.041940f},
|
||||||
{-0.023498f, 0.093664f, 0.025543f},
|
{-0.023498f, 0.093664f, 0.025543f},
|
||||||
{0.082899f, 0.048320f, 0.007491f},
|
{0.082899f, 0.048320f, 0.007491f},
|
||||||
{0.075712f, 0.074139f, 0.081965f},
|
{0.075712f, 0.074139f, 0.081965f},
|
||||||
{-0.143501f, 0.018263f, -0.136138f},
|
{-0.143501f, 0.018263f, -0.136138f},
|
||||||
{-0.025767f, -0.082035f, -0.040023f},
|
{-0.025767f, -0.082035f, -0.040023f},
|
||||||
{-0.111849f, -0.055589f, -0.032361f}};
|
{-0.111849f, -0.055589f, -0.032361f}};
|
||||||
float flux_latent_rgb_bias[3] = {0.024600f, -0.006937f, -0.008089f};
|
float flux_latent_rgb_bias[3] = {0.024600f, -0.006937f, -0.008089f};
|
||||||
|
|
||||||
const float flux2_latent_rgb_proj[32][3] = {
|
const float flux2_latent_rgb_proj[32][3] = {
|
||||||
{0.000736f, -0.008385f, -0.019710f},
|
{0.000736f, -0.008385f, -0.019710f},
|
||||||
{-0.001352f, -0.016392f, 0.020693f},
|
{-0.001352f, -0.016392f, 0.020693f},
|
||||||
{-0.006376f, 0.002428f, 0.036736f},
|
{-0.006376f, 0.002428f, 0.036736f},
|
||||||
{0.039384f, 0.074167f, 0.119789f},
|
{0.039384f, 0.074167f, 0.119789f},
|
||||||
{0.007464f, -0.005705f, -0.004734f},
|
{0.007464f, -0.005705f, -0.004734f},
|
||||||
{-0.004086f, 0.005287f, -0.000409f},
|
{-0.004086f, 0.005287f, -0.000409f},
|
||||||
{-0.032835f, 0.050802f, -0.028120f},
|
{-0.032835f, 0.050802f, -0.028120f},
|
||||||
{-0.003158f, -0.000835f, 0.000406f},
|
{-0.003158f, -0.000835f, 0.000406f},
|
||||||
{-0.112840f, -0.084337f, -0.023083f},
|
{-0.112840f, -0.084337f, -0.023083f},
|
||||||
{0.001462f, -0.006656f, 0.000549f},
|
{0.001462f, -0.006656f, 0.000549f},
|
||||||
{-0.009980f, -0.007480f, 0.009702f},
|
{-0.009980f, -0.007480f, 0.009702f},
|
||||||
{0.032540f, 0.000214f, -0.061388f},
|
{0.032540f, 0.000214f, -0.061388f},
|
||||||
{0.011023f, 0.000694f, 0.007143f},
|
{0.011023f, 0.000694f, 0.007143f},
|
||||||
{-0.001468f, -0.006723f, -0.001678f},
|
{-0.001468f, -0.006723f, -0.001678f},
|
||||||
{-0.005921f, -0.010320f, -0.003907f},
|
{-0.005921f, -0.010320f, -0.003907f},
|
||||||
{-0.028434f, 0.027584f, 0.018457f},
|
{-0.028434f, 0.027584f, 0.018457f},
|
||||||
{0.014349f, 0.011523f, 0.000441f},
|
{0.014349f, 0.011523f, 0.000441f},
|
||||||
{0.009874f, 0.003081f, 0.001507f},
|
{0.009874f, 0.003081f, 0.001507f},
|
||||||
{0.002218f, 0.005712f, 0.001563f},
|
{0.002218f, 0.005712f, 0.001563f},
|
||||||
{0.053010f, -0.019844f, 0.008683f},
|
{0.053010f, -0.019844f, 0.008683f},
|
||||||
{-0.002507f, 0.005384f, 0.000938f},
|
{-0.002507f, 0.005384f, 0.000938f},
|
||||||
{-0.002177f, -0.011366f, 0.003559f},
|
{-0.002177f, -0.011366f, 0.003559f},
|
||||||
{-0.000261f, 0.015121f, -0.003240f},
|
{-0.000261f, 0.015121f, -0.003240f},
|
||||||
{-0.003944f, -0.002083f, 0.005043f},
|
{-0.003944f, -0.002083f, 0.005043f},
|
||||||
{-0.009138f, 0.011336f, 0.003781f},
|
{-0.009138f, 0.011336f, 0.003781f},
|
||||||
{0.011429f, 0.003985f, -0.003855f},
|
{0.011429f, 0.003985f, -0.003855f},
|
||||||
{0.010518f, -0.005586f, 0.010131f},
|
{0.010518f, -0.005586f, 0.010131f},
|
||||||
{0.007883f, 0.002912f, -0.001473f},
|
{0.007883f, 0.002912f, -0.001473f},
|
||||||
{-0.003318f, -0.003160f, 0.003684f},
|
{-0.003318f, -0.003160f, 0.003684f},
|
||||||
{-0.034560f, -0.008740f, 0.012996f},
|
{-0.034560f, -0.008740f, 0.012996f},
|
||||||
{0.000166f, 0.001079f, -0.012153f},
|
{0.000166f, 0.001079f, -0.012153f},
|
||||||
{0.017772f, 0.000937f, -0.011953f}};
|
{0.017772f, 0.000937f, -0.011953f}};
|
||||||
float flux2_latent_rgb_bias[3] = {-0.028738f, -0.098463f, -0.107619f};
|
float flux2_latent_rgb_bias[3] = {-0.028738f, -0.098463f, -0.107619f};
|
||||||
|
|
||||||
// This one was taken straight from
|
// This one was taken straight from
|
||||||
// https://github.com/Stability-AI/sd3.5/blob/8565799a3b41eb0c7ba976d18375f0f753f56402/sd3_impls.py#L288-L303
|
// https://github.com/Stability-AI/sd3.5/blob/8565799a3b41eb0c7ba976d18375f0f753f56402/sd3_impls.py#L288-L303
|
||||||
// (MiT Licence)
|
// (MiT Licence)
|
||||||
const float sd3_latent_rgb_proj[16][3] = {
|
const float sd3_latent_rgb_proj[16][3] = {
|
||||||
{-0.0645f, 0.0177f, 0.1052f},
|
{-0.0645f, 0.0177f, 0.1052f},
|
||||||
{0.0028f, 0.0312f, 0.0650f},
|
{0.0028f, 0.0312f, 0.0650f},
|
||||||
{0.1848f, 0.0762f, 0.0360f},
|
{0.1848f, 0.0762f, 0.0360f},
|
||||||
{0.0944f, 0.0360f, 0.0889f},
|
{0.0944f, 0.0360f, 0.0889f},
|
||||||
{0.0897f, 0.0506f, -0.0364f},
|
{0.0897f, 0.0506f, -0.0364f},
|
||||||
{-0.0020f, 0.1203f, 0.0284f},
|
{-0.0020f, 0.1203f, 0.0284f},
|
||||||
{0.0855f, 0.0118f, 0.0283f},
|
{0.0855f, 0.0118f, 0.0283f},
|
||||||
{-0.0539f, 0.0658f, 0.1047f},
|
{-0.0539f, 0.0658f, 0.1047f},
|
||||||
{-0.0057f, 0.0116f, 0.0700f},
|
{-0.0057f, 0.0116f, 0.0700f},
|
||||||
{-0.0412f, 0.0281f, -0.0039f},
|
{-0.0412f, 0.0281f, -0.0039f},
|
||||||
{0.1106f, 0.1171f, 0.1220f},
|
{0.1106f, 0.1171f, 0.1220f},
|
||||||
{-0.0248f, 0.0682f, -0.0481f},
|
{-0.0248f, 0.0682f, -0.0481f},
|
||||||
{0.0815f, 0.0846f, 0.1207f},
|
{0.0815f, 0.0846f, 0.1207f},
|
||||||
{-0.0120f, -0.0055f, -0.0867f},
|
{-0.0120f, -0.0055f, -0.0867f},
|
||||||
{-0.0749f, -0.0634f, -0.0456f},
|
{-0.0749f, -0.0634f, -0.0456f},
|
||||||
{-0.1418f, -0.1457f, -0.1259f},
|
{-0.1418f, -0.1457f, -0.1259f},
|
||||||
};
|
};
|
||||||
float sd3_latent_rgb_bias[3] = {0, 0, 0};
|
float sd3_latent_rgb_bias[3] = {0, 0, 0};
|
||||||
|
|
||||||
const float sdxl_latent_rgb_proj[4][3] = {
|
const float sdxl_latent_rgb_proj[4][3] = {
|
||||||
{0.258303f, 0.277640f, 0.329699f},
|
{0.258303f, 0.277640f, 0.329699f},
|
||||||
{-0.299701f, 0.105446f, 0.014194f},
|
{-0.299701f, 0.105446f, 0.014194f},
|
||||||
{0.050522f, 0.186163f, -0.143257f},
|
{0.050522f, 0.186163f, -0.143257f},
|
||||||
{-0.211938f, -0.149892f, -0.080036f}};
|
{-0.211938f, -0.149892f, -0.080036f}};
|
||||||
float sdxl_latent_rgb_bias[3] = {0.144381f, -0.033313f, 0.007061f};
|
float sdxl_latent_rgb_bias[3] = {0.144381f, -0.033313f, 0.007061f};
|
||||||
|
|
||||||
const float sd_latent_rgb_proj[4][3] = {
|
const float sd_latent_rgb_proj[4][3] = {
|
||||||
{0.337366f, 0.216344f, 0.257386f},
|
{0.337366f, 0.216344f, 0.257386f},
|
||||||
{0.165636f, 0.386828f, 0.046994f},
|
{0.165636f, 0.386828f, 0.046994f},
|
||||||
{-0.267803f, 0.237036f, 0.223517f},
|
{-0.267803f, 0.237036f, 0.223517f},
|
||||||
{-0.178022f, -0.200862f, -0.678514f}};
|
{-0.178022f, -0.200862f, -0.678514f}};
|
||||||
float sd_latent_rgb_bias[3] = {-0.017478f, -0.055834f, -0.105825f};
|
float sd_latent_rgb_bias[3] = {-0.017478f, -0.055834f, -0.105825f};
|
||||||
|
|
||||||
void preview_latent_video(uint8_t* buffer, struct ggml_tensor* latents, const float (*latent_rgb_proj)[3], const float latent_rgb_bias[3], int patch_size) {
|
void preview_latent_video(uint8_t* buffer, struct ggml_tensor* latents, const float (*latent_rgb_proj)[3], const float latent_rgb_bias[3], int patch_size) {
|
||||||
size_t buffer_head = 0;
|
size_t buffer_head = 0;
|
||||||
|
|
||||||
uint32_t latent_width = static_cast<uint32_t>(latents->ne[0]);
|
uint32_t latent_width = static_cast<uint32_t>(latents->ne[0]);
|
||||||
uint32_t latent_height = static_cast<uint32_t>(latents->ne[1]);
|
uint32_t latent_height = static_cast<uint32_t>(latents->ne[1]);
|
||||||
uint32_t dim = static_cast<uint32_t>(latents->ne[ggml_n_dims(latents) - 1]);
|
uint32_t dim = static_cast<uint32_t>(latents->ne[ggml_n_dims(latents) - 1]);
|
||||||
uint32_t frames = 1;
|
uint32_t frames = 1;
|
||||||
if (ggml_n_dims(latents) == 4) {
|
if (ggml_n_dims(latents) == 4) {
|
||||||
frames = static_cast<uint32_t>(latents->ne[2]);
|
frames = static_cast<uint32_t>(latents->ne[2]);
|
||||||
}
|
}
|
||||||
|
|
||||||
uint32_t rgb_width = latent_width * patch_size;
|
uint32_t rgb_width = latent_width * patch_size;
|
||||||
uint32_t rgb_height = latent_height * patch_size;
|
uint32_t rgb_height = latent_height * patch_size;
|
||||||
|
|
||||||
uint32_t unpatched_dim = dim / (patch_size * patch_size);
|
uint32_t unpatched_dim = dim / (patch_size * patch_size);
|
||||||
|
|
||||||
for (uint32_t k = 0; k < frames; k++) {
|
for (uint32_t k = 0; k < frames; k++) {
|
||||||
for (uint32_t rgb_x = 0; rgb_x < rgb_width; rgb_x++) {
|
for (uint32_t rgb_x = 0; rgb_x < rgb_width; rgb_x++) {
|
||||||
for (uint32_t rgb_y = 0; rgb_y < rgb_height; rgb_y++) {
|
for (uint32_t rgb_y = 0; rgb_y < rgb_height; rgb_y++) {
|
||||||
int latent_x = rgb_x / patch_size;
|
int latent_x = rgb_x / patch_size;
|
||||||
int latent_y = rgb_y / patch_size;
|
int latent_y = rgb_y / patch_size;
|
||||||
|
|
||||||
int channel_offset = 0;
|
int channel_offset = 0;
|
||||||
if (patch_size > 1) {
|
if (patch_size > 1) {
|
||||||
channel_offset = ((rgb_y % patch_size) * patch_size + (rgb_x % patch_size));
|
channel_offset = ((rgb_y % patch_size) * patch_size + (rgb_x % patch_size));
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t latent_id = (latent_x * latents->nb[0] + latent_y * latents->nb[1] + k * latents->nb[2]);
|
size_t latent_id = (latent_x * latents->nb[0] + latent_y * latents->nb[1] + k * latents->nb[2]);
|
||||||
|
|
||||||
// should be incremented by 1 for each pixel
|
// should be incremented by 1 for each pixel
|
||||||
size_t pixel_id = k * rgb_width * rgb_height + rgb_y * rgb_width + rgb_x;
|
size_t pixel_id = k * rgb_width * rgb_height + rgb_y * rgb_width + rgb_x;
|
||||||
|
|
||||||
float r = 0, g = 0, b = 0;
|
float r = 0, g = 0, b = 0;
|
||||||
if (latent_rgb_proj != nullptr) {
|
if (latent_rgb_proj != nullptr) {
|
||||||
for (uint32_t d = 0; d < unpatched_dim; d++) {
|
for (uint32_t d = 0; d < unpatched_dim; d++) {
|
||||||
float value = *(float*)((char*)latents->data + latent_id + (d * patch_size * patch_size + channel_offset) * latents->nb[ggml_n_dims(latents) - 1]);
|
float value = *(float*)((char*)latents->data + latent_id + (d * patch_size * patch_size + channel_offset) * latents->nb[ggml_n_dims(latents) - 1]);
|
||||||
r += value * latent_rgb_proj[d][0];
|
r += value * latent_rgb_proj[d][0];
|
||||||
g += value * latent_rgb_proj[d][1];
|
g += value * latent_rgb_proj[d][1];
|
||||||
b += value * latent_rgb_proj[d][2];
|
b += value * latent_rgb_proj[d][2];
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// interpret first 3 channels as RGB
|
// interpret first 3 channels as RGB
|
||||||
r = *(float*)((char*)latents->data + latent_id + 0 * latents->nb[ggml_n_dims(latents) - 1]);
|
r = *(float*)((char*)latents->data + latent_id + 0 * latents->nb[ggml_n_dims(latents) - 1]);
|
||||||
g = *(float*)((char*)latents->data + latent_id + 1 * latents->nb[ggml_n_dims(latents) - 1]);
|
g = *(float*)((char*)latents->data + latent_id + 1 * latents->nb[ggml_n_dims(latents) - 1]);
|
||||||
b = *(float*)((char*)latents->data + latent_id + 2 * latents->nb[ggml_n_dims(latents) - 1]);
|
b = *(float*)((char*)latents->data + latent_id + 2 * latents->nb[ggml_n_dims(latents) - 1]);
|
||||||
}
|
}
|
||||||
if (latent_rgb_bias != nullptr) {
|
if (latent_rgb_bias != nullptr) {
|
||||||
// bias
|
// bias
|
||||||
r += latent_rgb_bias[0];
|
r += latent_rgb_bias[0];
|
||||||
g += latent_rgb_bias[1];
|
g += latent_rgb_bias[1];
|
||||||
b += latent_rgb_bias[2];
|
b += latent_rgb_bias[2];
|
||||||
}
|
}
|
||||||
// change range
|
// change range
|
||||||
r = r * .5f + .5f;
|
r = r * .5f + .5f;
|
||||||
g = g * .5f + .5f;
|
g = g * .5f + .5f;
|
||||||
b = b * .5f + .5f;
|
b = b * .5f + .5f;
|
||||||
|
|
||||||
// clamp rgb values to [0,1] range
|
// clamp rgb values to [0,1] range
|
||||||
r = r >= 0 ? r <= 1 ? r : 1 : 0;
|
r = r >= 0 ? r <= 1 ? r : 1 : 0;
|
||||||
g = g >= 0 ? g <= 1 ? g : 1 : 0;
|
g = g >= 0 ? g <= 1 ? g : 1 : 0;
|
||||||
b = b >= 0 ? b <= 1 ? b : 1 : 0;
|
b = b >= 0 ? b <= 1 ? b : 1 : 0;
|
||||||
|
|
||||||
buffer[pixel_id * 3 + 0] = (uint8_t)(r * 255);
|
buffer[pixel_id * 3 + 0] = (uint8_t)(r * 255);
|
||||||
buffer[pixel_id * 3 + 1] = (uint8_t)(g * 255);
|
buffer[pixel_id * 3 + 1] = (uint8_t)(g * 255);
|
||||||
buffer[pixel_id * 3 + 2] = (uint8_t)(b * 255);
|
buffer[pixel_id * 3 + 2] = (uint8_t)(b * 255);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
Reference in new issue
Block a user