evan/ai-toolkit
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added initial support for fine tuning and captioning with Qwen 3.8 27B

Browse Source
This commit is contained in:
Jaret Burkett committed 2026-10-18 12:25:28 -06:00
1 parent 8cf43dd2cc
commit e72468b456
17 files changed
+2770 -12
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

ui/src/app/jobs/new/options.tsx
+1
View File
@@ -151,6 +151,7 @@ export const quantizationOptions: SelectOption[] = [
{ value: 'uint4', label: '4 bit' },
{ value: 'uint3', label: '3 bit' },
{ value: 'uint2', label: '2 bit' },
{ value: 'convrotw4a8', label: '4bit convrot W4A8 (int4 weights, int8 activations)' },
];
export const defaultQtype = 'qfloat8';
ui/src/components/CaptionSimpleJob.tsx
+67
View File
@@ -20,7 +20,9 @@ import {
groupedCaptionerTypes,
maxNewTokensOptions,
maxResOptions,
mtpDepthOptions,
quantizationOptions,
videoFpsOptions,
} from '@/helpers/captionOptions';
type Props = {
@@ -243,6 +245,21 @@ const CaptionSimpleJob: React.FC<Props> = ({ jobConfig, setJobConfig, gpuIDs, se
/>
</div>
)}
{additionalSections.includes('caption.video_fps') && (
<div className="mt-4">
<SelectInput
label="Video FPS"
value={`${jobConfig.config.process[0].caption.video_fps ?? 2}`}
onChange={value => {
const num = parseFloat(value);
if (!isNaN(num)) {
setJobConfig(num, 'config.process[0].caption.video_fps');
}
}}
options={videoFpsOptions}
/>
</div>
)}
{additionalSections.includes('caption.batch_size') && (
<div className="mt-4">
<SelectInput
@@ -297,6 +314,44 @@ const CaptionSimpleJob: React.FC<Props> = ({ jobConfig, setJobConfig, gpuIDs, se
onChange={value => setJobConfig(value, 'config.process[0].caption.thinking')}
/>
)}
{additionalSections.includes('caption.mtp') && (
<>
<Checkbox
label="MTP Speculative Decoding"
checked={jobConfig.config.process[0].caption.mtp ?? true}
onChange={value => setJobConfig(value, 'config.process[0].caption.mtp')}
/>
{(jobConfig.config.process[0].caption.mtp ?? true) && (
<div className="pt-2">
<SelectInput
label="MTP Draft Depth"
value={`${jobConfig.config.process[0].caption.mtp_depth ?? 3}`}
onChange={value => {
const intVal = parseInt(value);
if (!isNaN(intVal)) {
setJobConfig(intVal, 'config.process[0].caption.mtp_depth');
}
}}
options={mtpDepthOptions}
/>
</div>
)}
</>
)}
{additionalSections.includes('caption.cuda_graph') && (
<Checkbox
label="CUDA Graph Decode"
checked={jobConfig.config.process[0].caption.cuda_graph ?? true}
onChange={value => setJobConfig(value, 'config.process[0].caption.cuda_graph')}
/>
)}
{additionalSections.includes('caption.vision_bf16') && (
<Checkbox
label="Vision Tower bf16"
checked={jobConfig.config.process[0].caption.vision_bf16 ?? true}
onChange={value => setJobConfig(value, 'config.process[0].caption.vision_bf16')}
/>
)}
{additionalSections.includes('caption.layer_offloading') && (
<>
<Checkbox
@@ -346,6 +401,18 @@ const CaptionSimpleJob: React.FC<Props> = ({ jobConfig, setJobConfig, gpuIDs, se
/>
</div>
)}
{additionalSections.includes('caption.system_prompt') && (
<div className="mb-4">
<TextInput
label="System Prompt"
value={jobConfig.config.process[0].caption.system_prompt || ''}
onChange={value => {
setJobConfig(value?.trim() === '' ? undefined : value, 'config.process[0].caption.system_prompt');
}}
placeholder="Optional system message"
/>
</div>
)}
<TextAreaInput
label="Caption Prompt"
value={jobConfig.config.process[0].caption.caption_prompt || ''}
ui/src/helpers/captionOptions.ts
+64 -3
View File
@@ -1,7 +1,7 @@
import { CloudLora, GroupedSelectOption, SelectOption } from "@/types";
type CaptionGroup = 'image' | 'music' | 'video' | 'image/video/sound';
type AdditionalSections = 'caption.model_name_or_path2' | 'caption.caption_prompt' | 'caption.max_res' | 'caption.max_new_tokens' | 'caption.fixed_caption' | 'caption.caption_format' | 'caption.extract_vocals_before_transcribe' | 'caption.keep_timestamps' | 'caption.thinking' | 'caption.batch_size' | 'caption.layer_offloading';
type CaptionGroup = 'image' | 'music' | 'video' | 'image/video' | 'image/video/sound';
type AdditionalSections = 'caption.model_name_or_path2' | 'caption.caption_prompt' | 'caption.max_res' | 'caption.max_new_tokens' | 'caption.fixed_caption' | 'caption.caption_format' | 'caption.extract_vocals_before_transcribe' | 'caption.keep_timestamps' | 'caption.thinking' | 'caption.batch_size' | 'caption.layer_offloading' | 'caption.mtp' | 'caption.cuda_graph' | 'caption.vision_bf16' | 'caption.video_fps' | 'caption.system_prompt';
export interface CaptionOption {
name: string;
@@ -26,6 +26,8 @@ export interface CaptionOption {
const defaultNameOrPath = '';
export const defaultQtype = 'convrot8';
const extensionsAudio = ['mp3', 'wav', 'flac', 'ogg'];
const extensionsImage = ['jpg', 'jpeg', 'png', 'bmp', 'webp'];
const extensionsVideo = ['mp4', 'mov', 'webm', 'mkv', 'avi'];
@@ -247,6 +249,51 @@ export const captionerTypes: CaptionOption[] = [
'caption.batch_size',
],
},
{
name: 'Qwen38Captioner',
label: 'Qwen3.8 27B',
group: 'image/video',
supportsLoras: true,
defaults: {
'config.process[0].caption.loras': [[], undefined],
'config.process[0].caption.model_name_or_path': ['Comfy-Org/Qwen3.8-27B/text_encoders/qwen3.8_27b_w4a8.safetensors', defaultNameOrPath],
'config.process[0].caption.extensions': [[...extensionsVideo, ...extensionsImage], defaultExtensions],
'config.process[0].caption.caption_prompt': [defaultImageCaptionPrompt, undefined],
'config.process[0].caption.max_res': [1024, undefined],
'config.process[0].caption.max_new_tokens': [512, undefined],
'config.process[0].caption.mtp': [true, undefined],
'config.process[0].caption.mtp_depth': [3, undefined],
'config.process[0].caption.cuda_graph': [true, undefined],
'config.process[0].caption.vision_bf16': [true, undefined],
'config.process[0].caption.video_fps': [2, undefined],
'config.process[0].caption.thinking': [false, undefined],
// the ComfyUI file is already W4A8; quantize only applies to the HF repo
'config.process[0].caption.quantize': [true, true],
'config.process[0].caption.qtype': ['convrotw4a8', defaultQtype],
'config.process[0].caption.compile': [false, false],
},
name_or_path_options: [
{ value: 'Comfy-Org/Qwen3.8-27B/text_encoders/qwen3.8_27b_w4a8.safetensors', label: 'Comfy-Org/Qwen3.8-27B (W4A8)' },
{ value: 'Qwen/Qwen3.8-27B', label: 'Qwen/Qwen3.8-27B (bf16, quantized on load)' },
],
captionPrompts: {
'General (image)': defaultImageCaptionPrompt,
'General (video)': defaultVideoCaptionPrompt,
'MiniMax H4 T2V': minimaxT2VCaptionPrompt,
'MiniMax H4 Image': minimaxImageCaptionPrompt,
},
additionalSections: [
'caption.caption_prompt',
'caption.system_prompt',
'caption.max_res',
'caption.max_new_tokens',
'caption.video_fps',
'caption.thinking',
'caption.mtp',
'caption.cuda_graph',
'caption.vision_bf16',
],
},
{
name: 'Ideogram4Captioner',
label: 'Ideogram 4 Captioner',
@@ -310,6 +357,7 @@ export const quantizationOptions: SelectOption[] = [
{ value: 'uint4', label: '4 bit' },
{ value: 'uint3', label: '3 bit' },
{ value: 'uint2', label: '2 bit' },
{ value: 'convrotw4a8', label: '4bit convrot W4A8 (int4 weights, int8 activations)' },
];
// AceStepCaptioner output layouts (caption.caption_format)
@@ -347,4 +395,17 @@ export const maxNewTokensOptions: SelectOption[] = [
{ value: '8192', label: '8192' },
];
export const defaultQtype = 'convrot8';
export const videoFpsOptions: SelectOption[] = [
{ value: '1', label: '1' },
{ value: '2', label: '2 (default)' },
{ value: '4', label: '4' },
{ value: '8', label: '8' },
];
export const mtpDepthOptions: SelectOption[] = [
{ value: '1', label: '1' },
{ value: '2', label: '2' },
{ value: '3', label: '3 (default)' },
{ value: '4', label: '4' },
{ value: '5', label: '5' },
];
ui/src/types.ts
+7
View File
@@ -349,6 +349,13 @@ export interface CaptionProcessConfig {
layer_offloading?: boolean;
layer_offloading_percent?: number;
loras?: CaptionLora[];
// Qwen3.8 captioner
mtp?: boolean;
mtp_depth?: number;
cuda_graph?: boolean;
vision_bf16?: boolean;
video_fps?: number;
system_prompt?: string;
}
}