Added a converter back to ldm from diffusers for sdxl. Can finally get to training it properly

extensions_built_in/image_reference_slider_trainer/ImageReferenceSliderTrainerProcess.py

@@ -95,7 +95,7 @@ class ImageReferenceSliderTrainerProcess(BaseSDTrainProcess):
             if weight_jitter > 0.0:
                 jitter_list = random.uniform(-weight_jitter, weight_jitter)
                 network_pos_weight += jitter_list
-                network_neg_weight += jitter_list
+                network_neg_weight += (jitter_list * -1.0)
 
             # if items in network_weight list are tensors, convert them to floats