evan/ai-toolkit
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add support for training VLLMs, omni models, and LLMs

Browse Source
This commit is contained in:
Jaret Burkett committed 2026-09-15 17:47:02 -06:00
1 parent 426ccb4ec4
commit db8dbd68c2
21 files changed
+1794 -524
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

ui/src/app/api/jobs/[jobID]/samples/route.ts
+14 -3
View File
@@ -28,11 +28,22 @@ export async function GET(request: NextRequest, { params }: { params: { jobID: s
// find all img (png, jpg, jpeg) files in the samples folder. Thumbnails
// live in the hidden .thumbs subfolder (and partial writes in .tmp) — the
// isFile check keeps those directories out even if their names ever match.
const samples = (await fs.promises.readdir(samplesFolder, { withFileTypes: true }))
const names = (await fs.promises.readdir(samplesFolder, { withFileTypes: true }))
.filter(entry => entry.isFile())
.map(entry => entry.name)
.map(entry => entry.name);
const isMedia = (file: string) =>
file.endsWith('.png') ||
file.endsWith('.jpg') ||
file.endsWith('.jpeg') ||
file.endsWith('.webp') ||
file.endsWith('.mp4') ||
file.endsWith('mp3');
const mediaStems = new Set(names.filter(isMedia).map(file => file.slice(0, file.lastIndexOf('.'))));
const samples = names
.filter(file => {
return file.endsWith('.png') || file.endsWith('.jpg') || file.endsWith('.jpeg') || file.endsWith('.webp') || file.endsWith('.mp4') || file.endsWith('mp3') || file.endsWith('wav') || file.endsWith('flac') || file.endsWith('ogg');
if (isMedia(file)) return true;
// text samples (LLM models); a .txt next to a media sample is its prompt sidecar, not a sample
return file.endsWith('.txt') && !mediaStems.has(file.slice(0, -4));
})
.map(file => {
return path.join(samplesFolder, file);
ui/src/app/generate/page.tsx
+305 -62
View File
@@ -3,10 +3,30 @@
import React, { Suspense, useCallback, useEffect, useMemo, useRef, useState } from 'react';
import { useSearchParams } from 'next/navigation';
import { Button } from '@headlessui/react';
import { ChevronDown, ChevronLeft, ChevronRight, Loader2, OctagonX, Play, Plus, Square, Sparkles, Trash2, X } from 'lucide-react';
import {
ChevronDown,
ChevronLeft,
ChevronRight,
Loader2,
OctagonX,
Play,
Plus,
Square,
Sparkles,
Trash2,
X,
} from 'lucide-react';
import { openConfirm } from '@/components/ConfirmModal';
import { TopBar, MainContent } from '@/components/layout';
import { Checkbox, CreatableSelectInput, NumberInput, SelectInput, SliderInput, TextAreaInput, TextInput } from '@/components/formInputs';
import {
Checkbox,
CreatableSelectInput,
NumberInput,
SelectInput,
SliderInput,
TextAreaInput,
TextInput,
} from '@/components/formInputs';
import { apiClient } from '@/utils/api';
import { startJob, stopJob } from '@/utils/jobs';
import { startQueue } from '@/utils/queue';
@@ -20,7 +40,6 @@ import { modelArchs, getGenerateDefaults, GenerateDefaults } from '@/app/jobs/ne
import GenerateFooter from '@/components/generate/GenerateFooter';
import LoraBrowserModal, { LoraPick } from '@/components/generate/LoraBrowserModal';
interface EngineStatus {
running: boolean;
engine: { jobId: string; jobName: string; gpuIds: string; status: string; ready: boolean } | null;
@@ -90,7 +109,11 @@ function Card({
}) {
return (
<div className="bg-gray-900 rounded-xl border border-gray-800">
<button type="button" onClick={onToggle} className="w-full flex items-center justify-between gap-2 px-3 py-1.5 text-xs text-gray-200 font-semibold">
<button
type="button"
onClick={onToggle}
className="w-full flex items-center justify-between gap-2 px-3 py-1.5 text-xs text-gray-200 font-semibold"
>
<span className="flex items-baseline gap-2 min-w-0">
<span>{title}</span>
{subtitle && <span className="text-[11px] font-normal text-gray-500 truncate">{subtitle}</span>}
@@ -124,7 +147,11 @@ function GeneratePageInner() {
return () => clearInterval(t);
}, []);
const engineJobId = preferredJob || engineStatus?.engine?.jobId || null;
const proxy = useCallback((path: string) => `/api/inference/${path}${engineJobId ? `${path.includes('?') ? '&' : '?'}job=${engineJobId}` : ''}`, [engineJobId]);
const proxy = useCallback(
(path: string) =>
`/api/inference/${path}${engineJobId ? `${path.includes('?') ? '&' : '?'}job=${engineJobId}` : ''}`,
[engineJobId],
);
usePollLoop(
() =>
@@ -139,7 +166,10 @@ function GeneratePageInner() {
setStartingJobId(null);
} else if (!mine) {
// no longer queued/running: the launch failed (or was stopped)
const job = await apiClient.get('/api/jobs', { params: { id: startingJobId } }).then(r => r.data).catch(() => null);
const job = await apiClient
.get('/api/jobs', { params: { id: startingJobId } })
.then(r => r.data)
.catch(() => null);
if (!job || ['error', 'stopped', 'completed'].includes(job.status)) {
setStartingJobId(null);
alert(`The engine did not start${job?.info ? `: ${job.info}` : ''}. Check the job log in the Queue.`);
@@ -172,7 +202,9 @@ function GeneratePageInner() {
try {
const gpu = isMac() ? 'mps' : gpuId;
const name = `inference_engine_gpu${gpu}`;
const existing = await apiClient.get('/api/jobs', { params: { job_type: 'inference' } }).then(r => r.data.jobs || []);
const existing = await apiClient
.get('/api/jobs', { params: { job_type: 'inference' } })
.then(r => r.data.jobs || []);
const match = existing.find((j: any) => j.name === name);
const res = await apiClient.post('/api/jobs', {
id: match ? match.id : null,
@@ -206,7 +238,10 @@ function GeneratePageInner() {
let alive = true;
while (Date.now() < deadline) {
await new Promise(r => setTimeout(r, 750));
const st = await apiClient.get(`/api/inference/status?job=${id}`).then(r => r.data).catch(() => null);
const st = await apiClient
.get(`/api/inference/status?job=${id}`)
.then(r => r.data)
.catch(() => null);
alive = !!st?.engines?.some((e: any) => e.jobId === id && (e.ready || e.status === 'stopping'));
if (!alive) break;
}
@@ -222,7 +257,8 @@ function GeneratePageInner() {
// a row that is stopped but whose process is still alive shows as 'stopping' from the status route
const isStopping = !!stoppingJobId || engineStatus?.engine?.status === 'stopping';
const isStarting = !isStopping && (engineBusy || !!startingJobId || (!!engineStatus?.engine && !engineStatus.running));
const isStarting =
!isStopping && (engineBusy || !!startingJobId || (!!engineStatus?.engine && !engineStatus.running));
const stoppingForMs = stoppingSince ? now - stoppingSince : isStopping ? 999_999 : 0;
// cancel a start that is not getting anywhere: a queued job (no process)
// is just marked stopped; a launched one gets the normal stop
@@ -259,12 +295,18 @@ function GeneratePageInner() {
};
// ---- models: the same arch list the training UI uses (jobs/new/options.tsx) ----
const archs: GenerateDefaults[] = useMemo(() => modelArchs.map(getGenerateDefaults), []);
// text-generating archs have no Generate page path yet
const archs: GenerateDefaults[] = useMemo(
() => modelArchs.filter(a => a.group !== 'llm').map(getGenerateDefaults),
[],
);
const ready = !!engineStatus?.running;
const [arch, setArch] = useState<string>(persisted.arch || 'zimage');
const [model, setModel] = useState<{ [key: string]: any }>(persisted.model || {});
const [sample, setSample] = useState<{ [key: string]: any }>(persisted.sample || { prompt: '', negative_prompt: '', seed: -1 });
const [sample, setSample] = useState<{ [key: string]: any }>(
persisted.sample || { prompt: '', negative_prompt: '', seed: -1 },
);
const entry = useMemo(() => archs.find(a => a.arch === arch) || null, [archs, arch]);
const applyArch = (name: string) => {
@@ -272,7 +314,14 @@ function GeneratePageInner() {
const e = archs.find(a => a.arch === name);
if (!e) return;
setModel({ ...e.model });
setSample(s => ({ ...s, ...e.sample, prompt: s.prompt, negative_prompt: s.negative_prompt, seed: s.seed ?? -1, ctrl_img: undefined }));
setSample(s => ({
...s,
...e.sample,
prompt: s.prompt,
negative_prompt: s.negative_prompt,
seed: s.seed ?? -1,
ctrl_img: undefined,
}));
};
useEffect(() => {
if (archs.length && !Object.keys(model).length) applyArch(archs.some(a => a.arch === arch) ? arch : archs[0].arch);
@@ -652,7 +701,9 @@ function GeneratePageInner() {
: engineStatus?.engine
? `Engine ${engineStatus.engine.status}`
: 'Engine off';
const vram = health?.vram ? `${(health.vram.used / 1e9).toFixed(1)} / ${(health.vram.total / 1e9).toFixed(0)} GB` : null;
const vram = health?.vram
? `${(health.vram.used / 1e9).toFixed(1)} / ${(health.vram.total / 1e9).toFixed(0)} GB`
: null;
const pool = health?.pool;
return (
@@ -666,11 +717,18 @@ function GeneratePageInner() {
<div className="flex items-center gap-2 text-xs sm:text-sm">
{isStopping ? (
<>
<Button disabled className="px-2 py-1 rounded-md bg-red-900 text-white/70 flex items-center gap-1 cursor-wait">
<Button
disabled
className="px-2 py-1 rounded-md bg-red-900 text-white/70 flex items-center gap-1 cursor-wait"
>
<Loader2 className="w-4 h-4 animate-spin" /> Stopping engine…
</Button>
{stoppingForMs > 8000 && (
<Button onClick={forceStop} className="px-2 py-1 rounded-md bg-red-700 hover:bg-red-600 text-white flex items-center gap-1" title="Kill the engine process">
<Button
onClick={forceStop}
className="px-2 py-1 rounded-md bg-red-700 hover:bg-red-600 text-white flex items-center gap-1"
title="Kill the engine process"
>
<OctagonX className="w-4 h-4" /> Force stop
</Button>
)}
@@ -680,7 +738,11 @@ function GeneratePageInner() {
<span className="text-green-400">Engine running</span>
{health?.active?.arch && <span className="text-gray-400 hidden sm:inline">· {health.active.arch}</span>}
{vram && <span className="text-gray-400 hidden md:inline">· VRAM {vram}</span>}
<Button onClick={stopEngine} disabled={engineBusy} className="ml-2 px-2 py-1 rounded-md bg-red-700 hover:bg-red-600 disabled:opacity-60 text-white flex items-center gap-1">
<Button
onClick={stopEngine}
disabled={engineBusy}
className="ml-2 px-2 py-1 rounded-md bg-red-700 hover:bg-red-600 disabled:opacity-60 text-white flex items-center gap-1"
>
{engineBusy ? <Loader2 className="w-4 h-4 animate-spin" /> : <Square className="w-4 h-4" />} Stop
</Button>
</>
@@ -702,15 +764,28 @@ function GeneratePageInner() {
</p>
{isStarting ? (
<div className="flex flex-col gap-2">
<Button disabled className="w-full px-3 py-2 rounded-md bg-blue-900 text-white/70 flex items-center justify-center gap-2 cursor-wait">
<Button
disabled
className="w-full px-3 py-2 rounded-md bg-blue-900 text-white/70 flex items-center justify-center gap-2 cursor-wait"
>
<Loader2 className="w-4 h-4 animate-spin" />
{engineStatus?.engine?.status === 'running' ? 'Engine loading…' : engineStatus?.engine ? `Engine ${engineStatus.engine.status}…` : 'Starting engine…'}
{engineStatus?.engine?.status === 'running'
? 'Engine loading…'
: engineStatus?.engine
? `Engine ${engineStatus.engine.status}…`
: 'Starting engine…'}
</Button>
{engineStatus?.engine?.status === 'queued' && (
<div className="text-[11px] text-gray-500">Waiting for the GPU queue to pick the job up. If the queue is stopped, start it from the Queue page or cancel here.</div>
<div className="text-[11px] text-gray-500">
Waiting for the GPU queue to pick the job up. If the queue is stopped, start it from the Queue
page or cancel here.
</div>
)}
{startingSince !== null && now - startingSince > 5000 && (
<Button onClick={cancelStart} className="w-full px-3 py-1.5 rounded-md bg-gray-800 hover:bg-gray-700 text-gray-200 text-sm">
<Button
onClick={cancelStart}
className="w-full px-3 py-1.5 rounded-md bg-gray-800 hover:bg-gray-700 text-gray-200 text-sm"
>
Cancel
</Button>
)}
@@ -718,7 +793,11 @@ function GeneratePageInner() {
) : (
<div className="flex flex-col gap-3">
{!isMac() && (
<select value={gpuId} onChange={e => setGpuId(e.target.value)} className="w-full bg-gray-800 text-gray-200 rounded-md px-3 py-2 border border-gray-700">
<select
value={gpuId}
onChange={e => setGpuId(e.target.value)}
className="w-full bg-gray-800 text-gray-200 rounded-md px-3 py-2 border border-gray-700"
>
{gpuList.map(g => (
<option key={g.index} value={`${g.index}`}>
GPU #{g.index} {g.name}
@@ -726,7 +805,10 @@ function GeneratePageInner() {
))}
</select>
)}
<Button onClick={startEngine} className="w-full px-3 py-2 rounded-md bg-blue-700 hover:bg-blue-600 text-white flex items-center justify-center gap-2">
<Button
onClick={startEngine}
className="w-full px-3 py-2 rounded-md bg-blue-700 hover:bg-blue-600 text-white flex items-center justify-center gap-2"
>
<Play className="w-4 h-4" /> Start engine
</Button>
</div>
@@ -756,7 +838,10 @@ function GeneratePageInner() {
) : null}
<div className="flex-1" />
{running && (
<Button onClick={cancel} className="px-2 py-0.5 rounded bg-gray-800/80 hover:bg-gray-700 text-gray-200 text-xs">
<Button
onClick={cancel}
className="px-2 py-0.5 rounded bg-gray-800/80 hover:bg-gray-700 text-gray-200 text-xs"
>
Cancel
</Button>
)}
@@ -766,7 +851,10 @@ function GeneratePageInner() {
Engine busy: {health.current.arch}
{health.current.total_steps ? ` ${health.current.step}/${health.current.total_steps}` : ''}
</span>
<Button onClick={cancelBusy} className="px-2 py-0.5 rounded bg-gray-800/80 hover:bg-gray-700 text-gray-200 text-xs">
<Button
onClick={cancelBusy}
className="px-2 py-0.5 rounded bg-gray-800/80 hover:bg-gray-700 text-gray-200 text-xs"
>
Cancel
</Button>
</>
@@ -774,10 +862,17 @@ function GeneratePageInner() {
</div>
{progress?.total ? (
<div className="absolute top-0 left-0 right-0 h-0.5 bg-gray-800 z-10">
<div className="h-0.5 bg-blue-500 transition-all" style={{ width: `${Math.min(100, (progress.step / progress.total) * 100)}%` }} />
<div
className="h-0.5 bg-blue-500 transition-all"
style={{ width: `${Math.min(100, (progress.step / progress.total) * 100)}%` }}
/>
</div>
) : null}
{error && <div className="absolute bottom-3 left-3 right-3 z-10 text-rose-300 text-sm bg-rose-950/70 rounded px-3 py-2 whitespace-pre-wrap">{error}</div>}
{error && (
<div className="absolute bottom-3 left-3 right-3 z-10 text-rose-300 text-sm bg-rose-950/70 rounded px-3 py-2 whitespace-pre-wrap">
{error}
</div>
)}
{/* live latent preview: shown while generating (or when nothing is selected) */}
<div className={`absolute inset-3 flex items-center justify-center ${showPreview ? '' : 'hidden'}`}>
@@ -791,7 +886,14 @@ function GeneratePageInner() {
{!showPreview && selected && (
<div className="absolute inset-3 flex items-center justify-center">
{selected.kind === 'video' ? (
<video key={selected.path} src={resultUrl(selected)} controls autoPlay loop className="max-w-full max-h-full rounded" />
<video
key={selected.path}
src={resultUrl(selected)}
controls
autoPlay
loop
className="max-w-full max-h-full rounded"
/>
) : selected.kind === 'audio' ? (
<div className="w-full max-w-xl bg-gray-900 rounded-xl p-6 text-center">
<div className="text-gray-300 text-sm mb-4 truncate" title={selected.prompt}>
@@ -800,9 +902,19 @@ function GeneratePageInner() {
<audio key={selected.path} src={resultUrl(selected)} controls autoPlay className="w-full" />
</div>
) : (
<a href={resultUrl(selected)} target="_blank" rel="noreferrer" className="w-full h-full flex items-center justify-center">
<a
href={resultUrl(selected)}
target="_blank"
rel="noreferrer"
className="w-full h-full flex items-center justify-center"
>
{/* eslint-disable-next-line @next/next/no-img-element */}
<img key={selected.path} src={resultUrl(selected)} alt={selected.prompt} className="max-w-full max-h-full rounded object-contain" />
<img
key={selected.path}
src={resultUrl(selected)}
alt={selected.prompt}
className="max-w-full max-h-full rounded object-contain"
/>
</a>
)}
</div>
@@ -901,20 +1013,40 @@ function GeneratePageInner() {
>
{sidebarOpen ? <ChevronRight className="w-3.5 h-3.5" /> : <ChevronLeft className="w-3.5 h-3.5" />}
</button>
<div className={`h-full flex flex-col border-l border-gray-800 bg-gray-900/60 ${sidebarOpen ? '' : 'hidden'}`}>
<div
className={`h-full flex flex-col border-l border-gray-800 bg-gray-900/60 ${sidebarOpen ? '' : 'hidden'}`}
>
<div className="compact-form flex-1 min-h-0 overflow-y-auto p-2 space-y-2">
<Card title="Model" subtitle={entry?.label || arch} open={cardOpen('model')} onToggle={() => toggleCard('model')}>
<SelectInput label="Architecture" value={arch} onChange={v => applyArch(v as string)} options={archOptions} disabled={!ready} />
<Card
title="Model"
subtitle={entry?.label || arch}
open={cardOpen('model')}
onToggle={() => toggleCard('model')}
>
<SelectInput
label="Architecture"
value={arch}
onChange={v => applyArch(v as string)}
options={archOptions}
disabled={!ready}
/>
<CreatableSelectInput
label="Name or Path"
value={model.name_or_path || ''}
onChange={(v: string | null) => setModel(m => ({ ...m, name_or_path: v || '' }))}
options={entry?.model?.name_or_path ? [{ value: entry.model.name_or_path, label: entry.model.name_or_path }] : []}
options={
entry?.model?.name_or_path
? [{ value: entry.model.name_or_path, label: entry.model.name_or_path }]
: []
}
placeholder="hub repo, local folder, or .safetensors"
/>
{'extras_name_or_path' in (entry?.model || {}) && (
<TextInput label="Extras Name or Path" value={model.extras_name_or_path || ''} onChange={v => setModel(m => ({ ...m, extras_name_or_path: v }))} />
<TextInput
label="Extras Name or Path"
value={model.extras_name_or_path || ''}
onChange={v => setModel(m => ({ ...m, extras_name_or_path: v }))}
/>
)}
<div className="grid grid-cols-2 gap-2">
<SelectInput
@@ -931,8 +1063,16 @@ function GeneratePageInner() {
/>
</div>
<div className="grid grid-cols-2 gap-2 mt-2">
<Checkbox label="Low VRAM" checked={!!model.low_vram} onChange={v => setModel(m => ({ ...m, low_vram: v }))} />
<Checkbox label="Layer offloading" checked={!!model.layer_offloading} onChange={v => setModel(m => ({ ...m, layer_offloading: v }))} />
<Checkbox
label="Low VRAM"
checked={!!model.low_vram}
onChange={v => setModel(m => ({ ...m, low_vram: v }))}
/>
<Checkbox
label="Layer offloading"
checked={!!model.layer_offloading}
onChange={v => setModel(m => ({ ...m, layer_offloading: v }))}
/>
</div>
{model.layer_offloading && (
<div className="mt-2 space-y-1">
@@ -955,11 +1095,19 @@ function GeneratePageInner() {
</div>
)}
</Card>
<Card title="LoRAs" subtitle={loras.length ? `${activeLoras.length}/${loras.length} active` : 'none'} open={cardOpen('loras')} onToggle={() => toggleCard('loras')}>
<Card
title="LoRAs"
subtitle={loras.length ? `${activeLoras.length}/${loras.length} active` : 'none'}
open={cardOpen('loras')}
onToggle={() => toggleCard('loras')}
>
<div className="space-y-2">
{loras.map((l, i) => (
<div key={l.path} className={`bg-gray-950/60 border border-gray-800 rounded-md px-2 py-1.5 ${l.disabled ? 'opacity-50' : ''}`}>
<div
key={l.path}
className={`bg-gray-950/60 border border-gray-800 rounded-md px-2 py-1.5 ${l.disabled ? 'opacity-50' : ''}`}
>
<div className="flex items-center gap-2">
<button
type="button"
@@ -977,7 +1125,10 @@ function GeneratePageInner() {
className={`absolute top-0.5 left-0.5 w-3 h-3 rounded-full bg-white transition-transform ${l.disabled ? '' : 'translate-x-3'}`}
/>
</button>
<span className={`text-xs truncate flex-1 ${l.disabled ? 'text-gray-500 line-through' : 'text-gray-200'}`} title={l.path}>
<span
className={`text-xs truncate flex-1 ${l.disabled ? 'text-gray-500 line-through' : 'text-gray-200'}`}
title={l.path}
>
{l.name}
</span>
<input
@@ -1022,13 +1173,15 @@ function GeneratePageInner() {
description: (
<>
<p>
<strong>Off</strong> (default): each LoRA is applied dynamically as an extra term on the layers it targets. Strength
changes take effect on the next generation and adding or removing a LoRA never reloads the model.
<strong>Off</strong> (default): each LoRA is applied dynamically as an extra term on the
layers it targets. Strength changes take effect on the next generation and adding or
removing a LoRA never reloads the model.
</p>
<p className="mt-2">
<strong>On</strong>: the LoRA deltas are added into the model weights. Quantized weights are dequantized, merged, and
re-quantized with stochastic rounding so small deltas are not rounded away. Merged weights cannot be un-merged, so
changing the LoRA set or strengths reloads the affected components. Slightly faster per step than the dynamic path.
<strong>On</strong>: the LoRA deltas are added into the model weights. Quantized weights
are dequantized, merged, and re-quantized with stochastic rounding so small deltas are not
rounded away. Merged weights cannot be un-merged, so changing the LoRA set or strengths
reloads the affected components. Slightly faster per step than the dynamic path.
</p>
</>
),
@@ -1041,40 +1194,125 @@ function GeneratePageInner() {
// tagged prompt archs (audio): one field per tag, stored as the tagged prompt string
Object.entries(entry.sampleTags).map(([tagKey, tag]) => {
const tags = tagsToObj(sample.prompt || '');
const setTag = (v: any) => setSample(s => ({ ...s, prompt: objToTags({ ...tagsToObj(s.prompt || ''), [tagKey]: v }) }));
const setTag = (v: any) =>
setSample(s => ({ ...s, prompt: objToTags({ ...tagsToObj(s.prompt || ''), [tagKey]: v }) }));
const value = tags[tagKey] ?? '';
if (tag.type === 'multiline') return <TextAreaInput key={tagKey} label={tag.title} value={value} onChange={setTag} placeholder={`Enter ${tag.title.toLowerCase()}`} />;
if (tag.type === 'number') return <NumberInput key={tagKey} label={tag.title} value={value} onChange={setTag} placeholder={`Enter ${tag.title.toLowerCase()}`} />;
return <TextInput key={tagKey} label={tag.title} value={value} onChange={setTag} placeholder={`Enter ${tag.title.toLowerCase()}`} />;
if (tag.type === 'multiline')
return (
<TextAreaInput
key={tagKey}
label={tag.title}
value={value}
onChange={setTag}
placeholder={`Enter ${tag.title.toLowerCase()}`}
/>
);
if (tag.type === 'number')
return (
<NumberInput
key={tagKey}
label={tag.title}
value={value}
onChange={setTag}
placeholder={`Enter ${tag.title.toLowerCase()}`}
/>
);
return (
<TextInput
key={tagKey}
label={tag.title}
value={value}
onChange={setTag}
placeholder={`Enter ${tag.title.toLowerCase()}`}
/>
);
})
) : (
<TextAreaInput label="Prompt" value={sample.prompt || ''} onChange={v => setSample(s => ({ ...s, prompt: v }))} placeholder={modality === 'audio' ? 'song description, lyrics, bpm…' : 'describe what to generate'} />
<TextAreaInput
label="Prompt"
value={sample.prompt || ''}
onChange={v => setSample(s => ({ ...s, prompt: v }))}
placeholder={
modality === 'audio' ? 'song description, lyrics, bpm…' : 'describe what to generate'
}
/>
)}
{(sample.guidance_scale ?? 4) > 1 && modality !== 'audio' && (
<TextAreaInput label="Negative prompt" value={sample.negative_prompt || ''} onChange={v => setSample(s => ({ ...s, negative_prompt: v }))} />
<TextAreaInput
label="Negative prompt"
value={sample.negative_prompt || ''}
onChange={v => setSample(s => ({ ...s, negative_prompt: v }))}
/>
)}
<div className="grid grid-cols-3 gap-2">
{modality !== 'audio' && (
<>
<NumberInput label="Width" value={sample.width ?? 1024} onChange={v => setSample(s => ({ ...s, width: v }))} min={64} max={4096} />
<NumberInput label="Height" value={sample.height ?? 1024} onChange={v => setSample(s => ({ ...s, height: v }))} min={64} max={4096} />
<NumberInput
label="Width"
value={sample.width ?? 1024}
onChange={v => setSample(s => ({ ...s, width: v }))}
min={64}
max={4096}
/>
<NumberInput
label="Height"
value={sample.height ?? 1024}
onChange={v => setSample(s => ({ ...s, height: v }))}
min={64}
max={4096}
/>
</>
)}
<NumberInput label="Steps" value={sample.num_inference_steps ?? 25} onChange={v => setSample(s => ({ ...s, num_inference_steps: v }))} min={1} max={200} />
<NumberInput label="Guidance" value={sample.guidance_scale ?? 4} onChange={v => setSample(s => ({ ...s, guidance_scale: v }))} min={1} max={30} />
<NumberInput label="Seed (-1 random)" value={sample.seed ?? -1} onChange={v => setSample(s => ({ ...s, seed: v }))} min={-1} max={4294967295} />
<NumberInput
label="Steps"
value={sample.num_inference_steps ?? 25}
onChange={v => setSample(s => ({ ...s, num_inference_steps: v }))}
min={1}
max={200}
/>
<NumberInput
label="Guidance"
value={sample.guidance_scale ?? 4}
onChange={v => setSample(s => ({ ...s, guidance_scale: v }))}
min={1}
max={30}
/>
<NumberInput
label="Seed (-1 random)"
value={sample.seed ?? -1}
onChange={v => setSample(s => ({ ...s, seed: v }))}
min={-1}
max={4294967295}
/>
{modality === 'audio' && sample.duration !== undefined && (
<NumberInput label="Duration (s)" value={sample.duration} onChange={v => setSample(s => ({ ...s, duration: v }))} min={1} max={600} />
<NumberInput
label="Duration (s)"
value={sample.duration}
onChange={v => setSample(s => ({ ...s, duration: v }))}
min={1}
max={600}
/>
)}
{modality === 'video' && (
<>
<NumberInput label="Frames" value={sample.num_frames ?? 33} onChange={v => setSample(s => ({ ...s, num_frames: v }))} min={1} max={1000} />
<NumberInput label="FPS" value={sample.fps ?? 16} onChange={v => setSample(s => ({ ...s, fps: v }))} min={1} max={60} />
<NumberInput
label="Frames"
value={sample.num_frames ?? 33}
onChange={v => setSample(s => ({ ...s, num_frames: v }))}
min={1}
max={1000}
/>
<NumberInput
label="FPS"
value={sample.fps ?? 16}
onChange={v => setSample(s => ({ ...s, fps: v }))}
min={1}
max={60}
/>
</>
)}
</div>
</Card>
</div>
{/* always-visible action bar */}
<div className="shrink-0 p-2 border-t border-gray-800 bg-gray-900 flex gap-2">
@@ -1096,7 +1334,12 @@ function GeneratePageInner() {
</div>
</div>
</MainContent>
<GenerateFooter jobId={engineJobId} status={footerStatus} busy={running || isStarting} progress={running ? progress : null} />
<GenerateFooter
jobId={engineJobId}
status={footerStatus}
busy={running || isStarting}
progress={running ? progress : null}
/>
<LoraBrowserModal isOpen={loraModalOpen} onClose={() => setLoraModalOpen(false)} onPick={addLora} />
</>
);
ui/src/app/jobs/new/SimpleJob.tsx
+407 -366
View File
@@ -82,6 +82,8 @@ export default function SimpleJob({
const isVideoModel = !!(modelArch?.group === 'video');
const isAudioModel = !!(modelArch?.group === 'audio');
// text-generating models: samples are media in, text out (no size)
const isLlmModel = !!(modelArch?.group === 'llm');
const taggedSampleArr: Record<string, any>[] | null = useMemo(() => {
if (!modelArch) return null;
@@ -213,6 +215,9 @@ export default function SimpleJob({
numDatasetCols -= 1;
numSampleTopCols -= 1;
}
if (isLlmModel) {
numSampleTopCols -= 1;
}
if (numDatasetCols == 3) {
datasetStyleClass = 'grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-6';
}
@@ -296,6 +301,18 @@ export default function SimpleJob({
placeholder=""
required
/>
{modelArch?.additionalSections?.includes('model.model_kwargs.instruction') && (
<TextAreaInput
label="LLM Prompt"
className="pt-2"
rows={2}
docKey="model.model_kwargs.instruction"
value={jobConfig.config.process[0].model.model_kwargs?.instruction ?? ''}
onChange={value => setJobConfig(value, 'config.process[0].model.model_kwargs.instruction')}
placeholder="Describe this in detail."
required
/>
)}
{modelArch?.additionalSections?.includes('model.assistant_lora_path') && (
<TextInput
label="Training Adapter Path"
@@ -489,7 +506,7 @@ export default function SimpleJob({
}}
options={transformerQuantizationOptions}
/>
{!disableSections.includes('model.quantize_te') && (
{!disableSections.includes('model.quantize_te') && !isLlmModel && (
<SelectInput
label="Text Encoder"
value={
@@ -507,25 +524,29 @@ export default function SimpleJob({
options={quantizationOptions}
/>
)}
<FormGroup label="Compile Options">
<></>
</FormGroup>
<Checkbox
label="Compile Model"
checked={jobConfig.config.process[0].model.compile || false}
onChange={value => {
setJobConfig(value, 'config.process[0].model.compile');
if (value) {
for (const key in defaultCompileOptions) {
setJobConfig((defaultCompileOptions as any)[key], `config.process[0].model.${key}`);
}
} else {
for (const key in defaultCompileOptions) {
setJobConfig(undefined, `config.process[0].model.${key}`);
}
}
}}
/>
{!isLlmModel && (
<>
<FormGroup label="Compile Options">
<></>
</FormGroup>
<Checkbox
label="Compile Model"
checked={jobConfig.config.process[0].model.compile || false}
onChange={value => {
setJobConfig(value, 'config.process[0].model.compile');
if (value) {
for (const key in defaultCompileOptions) {
setJobConfig((defaultCompileOptions as any)[key], `config.process[0].model.${key}`);
}
} else {
for (const key in defaultCompileOptions) {
setJobConfig(undefined, `config.process[0].model.${key}`);
}
}
}}
/>
</>
)}
</Card>
)}
{modelArch?.additionalSections?.includes('model.multistage') && (
@@ -737,56 +758,58 @@ export default function SimpleJob({
required
/>
</div>
<div>
{disableSections.includes('train.timestep_type') ? null : (
{!isLlmModel && (
<div>
{disableSections.includes('train.timestep_type') ? null : (
<SelectInput
label="Timestep Type"
value={jobConfig.config.process[0].train.timestep_type}
disabled={disableSections.includes('train.timestep_type') || false}
onChange={value => setJobConfig(value, 'config.process[0].train.timestep_type')}
options={[
{ value: 'sigmoid', label: 'Sigmoid' },
{ value: 'linear', label: 'Linear' },
{ value: 'shift', label: 'Shift' },
{ value: 'weighted', label: 'Weighted' },
]}
/>
)}
<SelectInput
label="Timestep Type"
value={jobConfig.config.process[0].train.timestep_type}
disabled={disableSections.includes('train.timestep_type') || false}
onChange={value => setJobConfig(value, 'config.process[0].train.timestep_type')}
label="Timestep Bias"
className="pt-2"
value={jobConfig.config.process[0].train.content_or_style}
onChange={value => setJobConfig(value, 'config.process[0].train.content_or_style')}
options={[
{ value: 'sigmoid', label: 'Sigmoid' },
{ value: 'linear', label: 'Linear' },
{ value: 'shift', label: 'Shift' },
{ value: 'weighted', label: 'Weighted' },
{ value: 'balanced', label: 'Balanced' },
{ value: 'content', label: 'High Noise' },
{ value: 'style', label: 'Low Noise' },
]}
/>
)}
<SelectInput
label="Timestep Bias"
className="pt-2"
value={jobConfig.config.process[0].train.content_or_style}
onChange={value => setJobConfig(value, 'config.process[0].train.content_or_style')}
options={[
{ value: 'balanced', label: 'Balanced' },
{ value: 'content', label: 'High Noise' },
{ value: 'style', label: 'Low Noise' },
]}
/>
<SelectInput
label="Loss Type"
className="pt-2"
value={jobConfig.config.process[0].train.loss_type}
onChange={value => setJobConfig(value, 'config.process[0].train.loss_type')}
options={[
{ value: 'mse', label: 'Mean Squared Error' },
{ value: 'mae', label: 'Mean Absolute Error' },
{ value: 'wavelet', label: 'Wavelet' },
{ value: 'stepped', label: 'Stepped Recovery' },
]}
/>
{modelArch?.additionalSections?.includes('train.audio_loss_multiplier') && (
<NumberInput
label="Audio Loss Multiplier"
<SelectInput
label="Loss Type"
className="pt-2"
value={jobConfig.config.process[0].train.audio_loss_multiplier ?? 1.0}
onChange={value => setJobConfig(value, 'config.process[0].train.audio_loss_multiplier')}
placeholder="eg. 1.0"
docKey={'train.audio_loss_multiplier'}
min={0}
value={jobConfig.config.process[0].train.loss_type}
onChange={value => setJobConfig(value, 'config.process[0].train.loss_type')}
options={[
{ value: 'mse', label: 'Mean Squared Error' },
{ value: 'mae', label: 'Mean Absolute Error' },
{ value: 'wavelet', label: 'Wavelet' },
{ value: 'stepped', label: 'Stepped Recovery' },
]}
/>
)}
</div>
{modelArch?.additionalSections?.includes('train.audio_loss_multiplier') && (
<NumberInput
label="Audio Loss Multiplier"
className="pt-2"
value={jobConfig.config.process[0].train.audio_loss_multiplier ?? 1.0}
onChange={value => setJobConfig(value, 'config.process[0].train.audio_loss_multiplier')}
placeholder="eg. 1.0"
docKey={'train.audio_loss_multiplier'}
min={0}
/>
)}
</div>
)}
<div>
<FormGroup label="EMA (Exponential Moving Average)">
<Checkbox
@@ -807,32 +830,34 @@ export default function SimpleJob({
/>
)}
<FormGroup label="Text Encoder Optimizations" className="pt-2">
{!disableSections.includes('train.unload_text_encoder') && (
{!isLlmModel && (
<FormGroup label="Text Encoder Optimizations" className="pt-2">
{!disableSections.includes('train.unload_text_encoder') && (
<Checkbox
label="Unload TE"
checked={jobConfig.config.process[0].train.unload_text_encoder || false}
docKey={'train.unload_text_encoder'}
onChange={value => {
setJobConfig(value, 'config.process[0].train.unload_text_encoder');
if (value) {
setJobConfig(false, 'config.process[0].train.cache_text_embeddings');
}
}}
/>
)}
<Checkbox
label="Unload TE"
checked={jobConfig.config.process[0].train.unload_text_encoder || false}
docKey={'train.unload_text_encoder'}
label="Cache Text Embeddings"
checked={jobConfig.config.process[0].train.cache_text_embeddings || false}
docKey={'train.cache_text_embeddings'}
onChange={value => {
setJobConfig(value, 'config.process[0].train.unload_text_encoder');
setJobConfig(value, 'config.process[0].train.cache_text_embeddings');
if (value) {
setJobConfig(false, 'config.process[0].train.cache_text_embeddings');
setJobConfig(false, 'config.process[0].train.unload_text_encoder');
}
}}
/>
)}
<Checkbox
label="Cache Text Embeddings"
checked={jobConfig.config.process[0].train.cache_text_embeddings || false}
docKey={'train.cache_text_embeddings'}
onChange={value => {
setJobConfig(value, 'config.process[0].train.cache_text_embeddings');
if (value) {
setJobConfig(false, 'config.process[0].train.unload_text_encoder');
}
}}
/>
</FormGroup>
</FormGroup>
)}
</div>
<div>
{disableSections.includes('train.diff_output_preservation') ||
@@ -914,212 +939,218 @@ export default function SimpleJob({
)}
</>
)}
<FormGroup label="Other" className="pt-2">
<>
<Checkbox
label="Contrastive Guidance Loss"
docKey={'train.do_guidance_loss'}
className="pt-1"
checked={jobConfig.config.process[0].train.do_guidance_loss || false}
onChange={value => {
if (value) {
setJobConfig(true, 'config.process[0].train.do_guidance_loss');
if (!jobConfig.config.process[0].train.guidance_loss_target) {
setJobConfig(4.0, 'config.process[0].train.guidance_loss_target');
{!isLlmModel && (
<FormGroup label="Other" className="pt-2">
<>
<Checkbox
label="Contrastive Guidance Loss"
docKey={'train.do_guidance_loss'}
className="pt-1"
checked={jobConfig.config.process[0].train.do_guidance_loss || false}
onChange={value => {
if (value) {
setJobConfig(true, 'config.process[0].train.do_guidance_loss');
if (!jobConfig.config.process[0].train.guidance_loss_target) {
setJobConfig(4.0, 'config.process[0].train.guidance_loss_target');
}
} else {
setJobConfig(undefined, 'config.process[0].train.do_guidance_loss');
setJobConfig(undefined, 'config.process[0].train.guidance_loss_target');
}
} else {
setJobConfig(undefined, 'config.process[0].train.do_guidance_loss');
setJobConfig(undefined, 'config.process[0].train.guidance_loss_target');
}
}}
/>
{jobConfig.config.process[0].train.do_guidance_loss && (
<>
<NumberInput
label="Guidance Loss Target"
docKey={'train.guidance_loss_target'}
value={(jobConfig.config.process[0].train.guidance_loss_target as number) || 4.0}
onChange={value => setJobConfig(value, 'config.process[0].train.guidance_loss_target')}
placeholder="eg. 3.0"
min={0}
/>
</>
)}
</>
</FormGroup>
</div>
</div>
</Card>
</div>
<div>
<Card
title="Validation"
toggled={!!validationConfig}
onToggle={value => {
if (value) {
setJobConfig(
{
validation_items: [{ image_path: '', prompt: '' }],
resolution: 1024,
validate_every_n_steps: 1,
validation_sigmas: [0.5],
},
'config.process[0].train.validation_config',
);
} else {
setJobConfig(undefined, 'config.process[0].train.validation_config');
}
}}
>
{validationConfig && (
<>
<p className="text-sm text-gray-400 mb-4">
Validation runs a stable loss check on a fixed set of images. Each image is encoded once at startup
and predicted at the selected sigmas with fixed seeds, so the result is always deterministic and
comparable across the run. The average loss is logged as val/loss every time validation runs. The
images need to match the concept of your dataset, but{' '}
<span className="font-bold text-gray-300">do not include the validation images in the dataset</span>.
They must be images containing the concept you want to train, but not an image trained on.
</p>
<div className="grid grid-cols-1 md:grid-cols-3 gap-6">
<NumberInput
label="Validate Every"
value={validationConfig.validate_every_n_steps}
onChange={value =>
setJobConfig(value, 'config.process[0].train.validation_config.validate_every_n_steps')
}
placeholder="eg. 10"
min={1}
required
/>
<NumberInput
label="Validation Resolution"
value={validationConfig.resolution}
onChange={value => setJobConfig(value, 'config.process[0].train.validation_config.resolution')}
placeholder="eg. 512"
min={64}
required
/>
<SelectInput
label="Validation Sigmas"
value={(validationConfig.validation_sigmas ?? [1.0, 0.75, 0.5, 0.25]).join(', ')}
onChange={value =>
setJobConfig(
value.split(',').map((v: string) => parseFloat(v)),
'config.process[0].train.validation_config.validation_sigmas',
)
}
options={[
{ value: '0.5', label: '0.5' },
{ value: '1, 0.5', label: '1.0, 0.5' },
{ value: '1, 0.66, 0.33', label: '1.0, 0.66, 0.33' },
{ value: '1, 0.75, 0.5, 0.25', label: '1.0, 0.75, 0.5, 0.25' },
]}
/>
</div>
<div className="mt-4">
<label className="block text-xs text-gray-300 mb-2">
Validation Images ({validationConfig.validation_items.length})
</label>
{validationConfig.validation_items.map((item, i) => (
<div key={i} className="rounded-lg pl-4 pr-1 py-3 mb-4 bg-gray-950">
<div className="flex items-center space-x-4">
<SampleControlImage
instruction="Add Image"
src={item.image_path === '' ? null : item.image_path}
onNewImageSelected={imagePath => {
setJobConfig(
imagePath ?? '',
`config.process[0].train.validation_config.validation_items[${i}].image_path`,
);
}}
/>
<div className="flex-1">
<TextInput
label="Prompt"
value={item.prompt}
onChange={value =>
setJobConfig(
value,
`config.process[0].train.validation_config.validation_items[${i}].prompt`,
)
}
placeholder="Enter prompt"
}}
/>
{jobConfig.config.process[0].train.do_guidance_loss && (
<>
<NumberInput
label="Guidance Loss Target"
docKey={'train.guidance_loss_target'}
value={(jobConfig.config.process[0].train.guidance_loss_target as number) || 4.0}
onChange={value => setJobConfig(value, 'config.process[0].train.guidance_loss_target')}
placeholder="eg. 3.0"
min={0}
/>
</div>
<div>
<button
type="button"
onClick={() =>
setJobConfig(
validationConfig.validation_items.filter((_, index) => index !== i),
'config.process[0].train.validation_config.validation_items',
)
}
className="rounded-full p-1 text-sm"
>
<X />
</button>
</div>
</div>
</div>
))}
<button
type="button"
onClick={() =>
setJobConfig(
[...validationConfig.validation_items, { image_path: '', prompt: '' }],
'config.process[0].train.validation_config.validation_items',
)
}
className="w-full px-4 py-2 bg-gray-700 hover:bg-gray-600 rounded-lg transition-colors"
>
Add Validation Image
</button>
</div>
</>
)}
</Card>
</div>
<div>
<Card title="Advanced" collapsible>
<div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-4 gap-6">
<div>
<Checkbox
label="Do Differential Guidance"
docKey={'train.do_differential_guidance'}
className="pt-1"
checked={jobConfig.config.process[0].train.do_differential_guidance || false}
onChange={value => {
let newValue = value == false ? undefined : value;
setJobConfig(newValue, 'config.process[0].train.do_differential_guidance');
if (!newValue) {
setJobConfig(undefined, 'config.process[0].train.differential_guidance_scale');
} else if (
jobConfig.config.process[0].train.differential_guidance_scale === undefined ||
jobConfig.config.process[0].train.differential_guidance_scale === null
) {
// set default differential guidance scale to 3.0
setJobConfig(3.0, 'config.process[0].train.differential_guidance_scale');
}
}}
/>
{jobConfig.config.process[0].train.differential_guidance_scale && (
<>
<NumberInput
label="Differential Guidance Scale"
className="pt-2"
value={(jobConfig.config.process[0].train.differential_guidance_scale as number) || 3.0}
onChange={value => setJobConfig(value, 'config.process[0].train.differential_guidance_scale')}
placeholder="eg. 3.0"
min={0}
/>
</>
</>
)}
</>
</FormGroup>
)}
</div>
</div>
</Card>
</div>
<div>
{!isLlmModel && (
<Card
title="Validation"
toggled={!!validationConfig}
onToggle={value => {
if (value) {
setJobConfig(
{
validation_items: [{ image_path: '', prompt: '' }],
resolution: 1024,
validate_every_n_steps: 1,
validation_sigmas: [0.5],
},
'config.process[0].train.validation_config',
);
} else {
setJobConfig(undefined, 'config.process[0].train.validation_config');
}
}}
>
{validationConfig && (
<>
<p className="text-sm text-gray-400 mb-4">
Validation runs a stable loss check on a fixed set of images. Each image is encoded once at startup
and predicted at the selected sigmas with fixed seeds, so the result is always deterministic and
comparable across the run. The average loss is logged as val/loss every time validation runs. The
images need to match the concept of your dataset, but{' '}
<span className="font-bold text-gray-300">do not include the validation images in the dataset</span>
. They must be images containing the concept you want to train, but not an image trained on.
</p>
<div className="grid grid-cols-1 md:grid-cols-3 gap-6">
<NumberInput
label="Validate Every"
value={validationConfig.validate_every_n_steps}
onChange={value =>
setJobConfig(value, 'config.process[0].train.validation_config.validate_every_n_steps')
}
placeholder="eg. 10"
min={1}
required
/>
<NumberInput
label="Validation Resolution"
value={validationConfig.resolution}
onChange={value => setJobConfig(value, 'config.process[0].train.validation_config.resolution')}
placeholder="eg. 512"
min={64}
required
/>
<SelectInput
label="Validation Sigmas"
value={(validationConfig.validation_sigmas ?? [1.0, 0.75, 0.5, 0.25]).join(', ')}
onChange={value =>
setJobConfig(
value.split(',').map((v: string) => parseFloat(v)),
'config.process[0].train.validation_config.validation_sigmas',
)
}
options={[
{ value: '0.5', label: '0.5' },
{ value: '1, 0.5', label: '1.0, 0.5' },
{ value: '1, 0.66, 0.33', label: '1.0, 0.66, 0.33' },
{ value: '1, 0.75, 0.5, 0.25', label: '1.0, 0.75, 0.5, 0.25' },
]}
/>
</div>
<div className="mt-4">
<label className="block text-xs text-gray-300 mb-2">
Validation Images ({validationConfig.validation_items.length})
</label>
{validationConfig.validation_items.map((item, i) => (
<div key={i} className="rounded-lg pl-4 pr-1 py-3 mb-4 bg-gray-950">
<div className="flex items-center space-x-4">
<SampleControlImage
instruction="Add Image"
src={item.image_path === '' ? null : item.image_path}
onNewImageSelected={imagePath => {
setJobConfig(
imagePath ?? '',
`config.process[0].train.validation_config.validation_items[${i}].image_path`,
);
}}
/>
<div className="flex-1">
<TextInput
label="Prompt"
value={item.prompt}
onChange={value =>
setJobConfig(
value,
`config.process[0].train.validation_config.validation_items[${i}].prompt`,
)
}
placeholder="Enter prompt"
/>
</div>
<div>
<button
type="button"
onClick={() =>
setJobConfig(
validationConfig.validation_items.filter((_, index) => index !== i),
'config.process[0].train.validation_config.validation_items',
)
}
className="rounded-full p-1 text-sm"
>
<X />
</button>
</div>
</div>
</div>
))}
<button
type="button"
onClick={() =>
setJobConfig(
[...validationConfig.validation_items, { image_path: '', prompt: '' }],
'config.process[0].train.validation_config.validation_items',
)
}
className="w-full px-4 py-2 bg-gray-700 hover:bg-gray-600 rounded-lg transition-colors"
>
Add Validation Image
</button>
</div>
</>
)}
</Card>
)}
</div>
<div>
{!isLlmModel && (
<Card title="Advanced" collapsible>
<div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-4 gap-6">
<div>
<Checkbox
label="Do Differential Guidance"
docKey={'train.do_differential_guidance'}
className="pt-1"
checked={jobConfig.config.process[0].train.do_differential_guidance || false}
onChange={value => {
let newValue = value == false ? undefined : value;
setJobConfig(newValue, 'config.process[0].train.do_differential_guidance');
if (!newValue) {
setJobConfig(undefined, 'config.process[0].train.differential_guidance_scale');
} else if (
jobConfig.config.process[0].train.differential_guidance_scale === undefined ||
jobConfig.config.process[0].train.differential_guidance_scale === null
) {
// set default differential guidance scale to 3.0
setJobConfig(3.0, 'config.process[0].train.differential_guidance_scale');
}
}}
/>
{jobConfig.config.process[0].train.differential_guidance_scale && (
<>
<NumberInput
label="Differential Guidance Scale"
className="pt-2"
value={(jobConfig.config.process[0].train.differential_guidance_scale as number) || 3.0}
onChange={value => setJobConfig(value, 'config.process[0].train.differential_guidance_scale')}
placeholder="eg. 3.0"
min={0}
/>
</>
)}
</div>
</div>
</Card>
)}
</div>
<div>
<Card title="Datasets">
<>
@@ -1359,7 +1390,7 @@ export default function SimpleJob({
/>
)}
</FormGroup>
{!isAudioModel && (
{!isAudioModel && !isLlmModel && (
<FormGroup label="Flipping" docKey={'datasets.flip'} className="mt-2">
<Checkbox
label={
@@ -1458,37 +1489,41 @@ export default function SimpleJob({
min={0}
required
/>
<SelectInput
label="Sampler"
className="pt-2"
value={jobConfig.config.process[0].sample.sampler}
onChange={value => setJobConfig(value, 'config.process[0].sample.sampler')}
options={[
{ value: 'flowmatch', label: 'FlowMatch' },
{ value: 'ddpm', label: 'DDPM' },
]}
/>
<NumberInput
label="Guidance Scale"
value={jobConfig.config.process[0].sample.guidance_scale}
onChange={value => setJobConfig(value, 'config.process[0].sample.guidance_scale')}
placeholder="eg. 1.0"
className="pt-2"
min={0}
required
/>
<NumberInput
label="Sample Steps"
value={jobConfig.config.process[0].sample.sample_steps}
onChange={value => setJobConfig(value, 'config.process[0].sample.sample_steps')}
placeholder="eg. 1"
className="pt-2"
min={1}
required
/>
{!isLlmModel && (
<>
<SelectInput
label="Sampler"
className="pt-2"
value={jobConfig.config.process[0].sample.sampler}
onChange={value => setJobConfig(value, 'config.process[0].sample.sampler')}
options={[
{ value: 'flowmatch', label: 'FlowMatch' },
{ value: 'ddpm', label: 'DDPM' },
]}
/>
<NumberInput
label="Guidance Scale"
value={jobConfig.config.process[0].sample.guidance_scale}
onChange={value => setJobConfig(value, 'config.process[0].sample.guidance_scale')}
placeholder="eg. 1.0"
className="pt-2"
min={0}
required
/>
<NumberInput
label="Sample Steps"
value={jobConfig.config.process[0].sample.sample_steps}
onChange={value => setJobConfig(value, 'config.process[0].sample.sample_steps')}
placeholder="eg. 1"
className="pt-2"
min={1}
required
/>
</>
)}
</div>
{!isAudioModel && (
{!isAudioModel && !isLlmModel && (
<div>
<NumberInput
label="Width"
@@ -1532,33 +1567,35 @@ export default function SimpleJob({
</div>
)}
<div>
<NumberInput
label="Seed"
value={jobConfig.config.process[0].sample.seed}
onChange={value => setJobConfig(value, 'config.process[0].sample.seed')}
placeholder="eg. 0"
min={0}
required
/>
{modelArch?.additionalSections?.includes('sample.duration') && (
{!isLlmModel && (
<div>
<NumberInput
label="Duration (seconds)"
value={jobConfig.config.process[0].sample.duration ?? 120}
onChange={value => setJobConfig(value, 'config.process[0].sample.duration')}
placeholder="eg. 120"
className="pt-2"
min={1}
label="Seed"
value={jobConfig.config.process[0].sample.seed}
onChange={value => setJobConfig(value, 'config.process[0].sample.seed')}
placeholder="eg. 0"
min={0}
required
/>
)}
<Checkbox
label="Walk Seed"
className="pt-4 pl-2"
checked={jobConfig.config.process[0].sample.walk_seed}
onChange={value => setJobConfig(value, 'config.process[0].sample.walk_seed')}
/>
</div>
{modelArch?.additionalSections?.includes('sample.duration') && (
<NumberInput
label="Duration (seconds)"
value={jobConfig.config.process[0].sample.duration ?? 120}
onChange={value => setJobConfig(value, 'config.process[0].sample.duration')}
placeholder="eg. 120"
className="pt-2"
min={1}
required
/>
)}
<Checkbox
label="Walk Seed"
className="pt-4 pl-2"
checked={jobConfig.config.process[0].sample.walk_seed}
onChange={value => setJobConfig(value, 'config.process[0].sample.walk_seed')}
/>
</div>
)}
<div>
<FormGroup label="Advanced Sampling" className="pt-2">
<div>
@@ -1713,8 +1750,8 @@ export default function SimpleJob({
label={`Prompt`}
value={sample.prompt}
onChange={value => setJobConfig(value, `config.process[0].sample.samples[${i}].prompt`)}
placeholder="Enter prompt"
required
placeholder={isLlmModel ? 'blank = LLM Prompt' : 'Enter prompt'}
required={!isLlmModel}
/>
)}
</>
@@ -1746,7 +1783,7 @@ export default function SimpleJob({
)}
<div className="grid w-full lg:grid-flow-col lg:auto-cols-fr gap-4 mt-2">
{!isAudioModel && (
{!isAudioModel && !isLlmModel && (
<TextInput
label={`Width`}
value={sample.width ? `${sample.width}` : ''}
@@ -1775,7 +1812,7 @@ export default function SimpleJob({
placeholder={`${jobConfig.config.process[0].sample.width} (default)`}
/>
)}
{!isAudioModel && (
{!isAudioModel && !isLlmModel && (
<TextInput
label={`Height`}
value={sample.height ? `${sample.height}` : ''}
@@ -1804,33 +1841,35 @@ export default function SimpleJob({
placeholder={`${jobConfig.config.process[0].sample.height} (default)`}
/>
)}
<TextInput
label={`Seed`}
value={sample.seed ? `${sample.seed}` : ''}
onChange={value => {
// remove any non-numeric characters
value = value.replace(/\D/g, '');
if (value === '') {
// remove the key from the config if empty
let newConfig = objectCopy(jobConfig);
if (newConfig.config.process[0].sample.samples[i]) {
delete newConfig.config.process[0].sample.samples[i].seed;
setJobConfig(
newConfig.config.process[0].sample.samples,
'config.process[0].sample.samples',
);
}
} else {
const intValue = parseInt(value);
if (!isNaN(intValue)) {
setJobConfig(intValue, `config.process[0].sample.samples[${i}].seed`);
{!isLlmModel && (
<TextInput
label={`Seed`}
value={sample.seed ? `${sample.seed}` : ''}
onChange={value => {
// remove any non-numeric characters
value = value.replace(/\D/g, '');
if (value === '') {
// remove the key from the config if empty
let newConfig = objectCopy(jobConfig);
if (newConfig.config.process[0].sample.samples[i]) {
delete newConfig.config.process[0].sample.samples[i].seed;
setJobConfig(
newConfig.config.process[0].sample.samples,
'config.process[0].sample.samples',
);
}
} else {
console.warn('Invalid seed value:', value);
const intValue = parseInt(value);
if (!isNaN(intValue)) {
setJobConfig(intValue, `config.process[0].sample.samples[${i}].seed`);
} else {
console.warn('Invalid seed value:', value);
}
}
}
}}
placeholder={`${jobConfig.config.process[0].sample.walk_seed ? jobConfig.config.process[0].sample.seed + i : jobConfig.config.process[0].sample.seed} (default)`}
/>
}}
placeholder={`${jobConfig.config.process[0].sample.walk_seed ? jobConfig.config.process[0].sample.seed + i : jobConfig.config.process[0].sample.seed} (default)`}
/>
)}
<TextInput
label={`LoRA Scale`}
value={sample.network_multiplier ? `${sample.network_multiplier}` : ''}
@@ -1883,6 +1922,8 @@ export default function SimpleJob({
{modelArch?.additionalSections?.includes('sample.ctrl_img') && (
<SampleControlImage
className="mt-6 ml-4"
instruction={isLlmModel ? 'Add Media' : undefined}
allowAudio={isLlmModel}
src={sample.ctrl_img}
onNewImageSelected={imagePath => {
if (!imagePath) {
ui/src/app/jobs/new/options.tsx
+63 -2
View File
@@ -7,6 +7,7 @@ import {
defaultSampleConfig,
defaultIdeogramSamplesConfig,
defaultYue2SampleConfig,
defaultQwen25OmniSampleConfig,
} from '@/helpers/defaultSamples';
type Control = 'depth' | 'line' | 'pose' | 'inpaint';
@@ -42,9 +43,10 @@ type AdditionalSections =
| 'model.assistant_lora_path'
| 'model.unconditional_lora_path'
| 'model.model_kwargs.kv_cache'
| 'model.model_kwargs.instruction'
| 'ideogram_4_prompt';
type ModelGroup = 'image' | 'instruction' | 'video' | 'experimental' | 'audio';
type ModelGroup = 'image' | 'instruction' | 'video' | 'experimental' | 'audio' | 'llm';
export interface CustomModelSelectOption {
label: string;
@@ -1378,6 +1380,62 @@ export const modelArchs: ModelArch[] = [
disableSections: ['network.conv'],
additionalSections: ['sample.multi_ctrl_imgs', 'model.low_vram', 'model.layer_offloading'],
},
{
name: 'qwen25_omni',
label: 'Qwen2.5-Omni',
group: 'llm',
defaults: {
// default updates when [selected, unselected] in the UI
'config.process[0].model.name_or_path': [
'ostris/qwen2_5_omni/text_encoders/qwen2_5_omni_7b_convrot8.safetensors',
defaultNameOrPath,
],
'config.process[0].model.quantize': [true, false],
'config.process[0].model.quantize_te': [false, false],
'config.process[0].model.low_vram': [false, false],
// the single-file thinker ships convrot8 layers; requesting convrot8 keeps them as-is
'config.process[0].model.qtype': ['convrot8', 'qfloat8'],
'config.process[0].train.unload_text_encoder': [false, false],
'config.process[0].train.noise_scheduler': ['flowmatch', 'flowmatch'],
'config.process[0].train.batch_size': [1, 1],
'config.process[0].sample': [defaultQwen25OmniSampleConfig, defaultSampleConfig],
// media is encoded on the GPU per step; the cache is optional and large (~54 MB per 300 s of audio)
'config.process[0].datasets[x].cache_latents_to_disk': [false, true],
'config.process[0].datasets[x].resolution': [[512], [512, 768, 1024]],
// the caption is the training target; a blank one trains nothing
'config.process[0].datasets[x].caption_dropout_rate': [0, 0.05],
'config.process[0].model.model_kwargs': [{ instruction: 'Describe this in detail.' }, {}],
},
disableSections: [
'network.conv',
'trigger_word',
'train.diff_output_preservation',
'train.blank_prompt_preservation',
'train.unload_text_encoder',
'slider',
],
additionalSections: ['model.model_kwargs.instruction', 'sample.ctrl_img', 'datasets.num_frames'],
modelNotes: (
<div className="space-y-2">
<p>
Qwen2.5-Omni 7B thinker as a text-generating model: audio, image or video in, text out. Each dataset item is a
media file (mp3, wav, flac, ogg, jpg, png, webp, mp4, ...) with a caption file next to it; the caption is the
text the model learns to produce for that media. One dataset can mix all three kinds. Video files are used
when Num Frames is above 1 and are seen as frames only (no audio track).
</p>
<p>
The instruction in <code>model_kwargs.instruction</code> is the user turn for every training item; use the
same wording when captioning with the trained LoRA. Samples take a media file per prompt and write the
generated text as a .txt file.
</p>
<p>
Media is encoded by the frozen audio and vision towers on the GPU each step, so Cache Latents to Disk can stay
off (the cache is about 54 MB per 300 seconds of audio). Batch size is 1. The LoRA trains the text stack only.
Watch <code>loss/ce</code>: next-token cross-entropy on the caption.
</p>
</div>
),
},
{
name: 'yue2',
label: 'YuE2',
@@ -1402,7 +1460,10 @@ export const modelArchs: ModelArch[] = [
'config.process[0].datasets[x].resolution': [[512], [512, 768, 1024]],
// blank captions break lyric following; the AR must always see the prefix
'config.process[0].datasets[x].caption_dropout_rate': [0, 0.05],
'config.process[0].model.model_kwargs': [{ cot: 'full', abc_dropout: 0.5, sample_ar_repetition_penalty: 1.2, ar_kl_weight: 0.2 }, {}],
'config.process[0].model.model_kwargs': [
{ cot: 'full', abc_dropout: 0.5, sample_ar_repetition_penalty: 1.2, ar_kl_weight: 0.2 },
{},
],
},
// native YuE2 prompt: style text, a [Lyrics] line, the lyrics
hasMultiLinePrompts: true,
ui/src/components/SampleControlImage.tsx
+9 -2
View File
@@ -9,13 +9,17 @@ import { encodeFilePathForUrl } from '@/utils/basic';
import type { AxiosProgressEvent } from 'axios';
const VIDEO_EXTS = ['.mp4', '.mov', '.webm', '.mkv', '.avi', '.m4v', '.wmv', '.flv'];
const AUDIO_EXTS = ['.mp3', '.wav', '.flac', '.ogg'];
const isVideoPath = (p: string) => VIDEO_EXTS.some(ext => p.toLowerCase().endsWith(ext));
const isAudioPath = (p: string) => AUDIO_EXTS.some(ext => p.toLowerCase().endsWith(ext));
interface Props {
src: string | null | undefined;
className?: string;
instruction?: string;
onNewImageSelected: (imagePath: string | null) => void;
/** also accept audio files (text-generating models take any media) */
allowAudio?: boolean;
}
export default function SampleControlImage({
@@ -23,6 +27,7 @@ export default function SampleControlImage({
className,
instruction = 'Add Control Image',
onNewImageSelected,
allowAudio = false,
}: Props) {
const [isUploading, setIsUploading] = useState(false);
const [uploadProgress, setUploadProgress] = useState(0);
@@ -31,7 +36,8 @@ export default function SampleControlImage({
const backgroundUrl = useMemo(() => {
if (localPreview) return localPreview;
// videos preview as a server-generated thumbnail
// videos preview as a server-generated thumbnail, audio as its waveform art
if (src && isAudioPath(src)) return `/api/audio/art/${encodeURIComponent(src)}`;
if (src) return `/api/img/${encodeFilePathForUrl(src)}${isVideoPath(src) ? '?thumb=1' : ''}`;
return null;
}, [src, localPreview]);
@@ -104,6 +110,7 @@ export default function SampleControlImage({
accept: {
'image/*': ['.png', '.jpg', '.jpeg', '.gif', '.bmp', '.webp'],
'video/*': VIDEO_EXTS,
...(allowAudio ? { 'audio/*': AUDIO_EXTS } : {}),
},
multiple: false,
noClick: true,
@@ -138,7 +145,7 @@ export default function SampleControlImage({
<input
ref={fileInputRef}
type="file"
accept="image/*,video/*"
accept={allowAudio ? 'image/*,video/*,audio/*' : 'image/*,video/*'}
className="hidden"
onChange={e => {
const file = e.currentTarget.files?.[0];
ui/src/components/SampleImageCard.tsx
+39 -8
View File
@@ -1,5 +1,5 @@
import React, { useRef, useEffect, useState, ReactNode } from 'react';
import { isVideo, isAudio, encodeFilePathForUrl } from '@/utils/basic';
import { isVideo, isAudio, isText, encodeFilePathForUrl } from '@/utils/basic';
interface SampleImageCardProps {
imageUrl: string;
@@ -39,6 +39,28 @@ const SampleImageCard: React.FC<SampleImageCardProps> = ({
const isItAudio = isAudio(imageUrl);
const isItVideo = isVideo(imageUrl);
const isItText = isText(imageUrl);
const [text, setText] = useState<string | null>(null);
// text samples: fetch the file body and render it in the card
useEffect(() => {
if (!isItText || !isVisible) return;
const controller = new AbortController();
fetch(`/api/img/${encodeFilePathForUrl(imageUrl)}`, { signal: controller.signal })
.then(r => (r.ok ? r.text() : Promise.reject(new Error(`HTTP ${r.status}`))))
.then(t => {
setText(t);
setLoaded(true);
})
.catch(err => {
if (err?.name !== 'AbortError') console.error('Sample text fetch failed:', err);
});
return () => {
controller.abort();
setText(null);
setLoaded(false);
};
}, [isItText, isVisible, imageUrl]);
// Observe both enter and exit
useEffect(() => {
@@ -69,7 +91,7 @@ const SampleImageCard: React.FC<SampleImageCardProps> = ({
// the element unmounts). A short debounce skips requests entirely during fast
// scrolls where the card is only briefly visible.
useEffect(() => {
if (isItAudio) return;
if (isItAudio || isItText) return;
if (!isVisible) return;
const controller = new AbortController();
@@ -114,7 +136,7 @@ const SampleImageCard: React.FC<SampleImageCardProps> = ({
setLoaded(false);
setVideoFallback(false);
};
}, [isVisible, isItAudio, isItVideo, imageUrl]);
}, [isVisible, isItAudio, isItText, isItVideo, imageUrl]);
return (
<div className={`flex flex-col ${className}`}>
@@ -127,14 +149,23 @@ const SampleImageCard: React.FC<SampleImageCardProps> = ({
onClick={onClick}
>
<div
className={`absolute rounded-t-lg shadow-md bg-gray-900 overflow-hidden transition-all duration-200 ${
className={`absolute rounded-t-lg shadow-md bg-gray-900 overflow-hidden transition-all duration-200 [container-type:inline-size] ${
selected ? 'inset-2' : 'inset-0'
} ${
isVisible && !isItAudio && !loaded ? 'animate-pulse' : ''
}`}
} ${isVisible && !isItAudio && !loaded ? 'animate-pulse' : ''}`}
>
{isVisible ? (
isItAudio ? (
isItText ? (
// font size scales with the card and the text length: a square of side W holds
// ~W^2 / (0.6 f^2) characters at size f, so f ~ 120/sqrt(n) cqw fills the box
<div
className="w-full h-full overflow-hidden p-[3cqw] leading-snug text-gray-200 whitespace-pre-wrap break-words text-left bg-gray-900"
style={{
fontSize: `clamp(2.5cqw, ${(120 / Math.sqrt(Math.max((text ?? '').length, 1))).toFixed(2)}cqw, 12cqw)`,
}}
>
{text ?? ''}
</div>
) : isItAudio ? (
<div className="w-full h-full flex items-center justify-center bg-gray-900">
<img
src={`/api/audio/art/${encodeURIComponent(imageUrl)}`}
ui/src/components/SampleImageViewer.tsx
+47 -13
View File
@@ -8,7 +8,7 @@ import classNames from 'classnames';
import { Menu, MenuButton, MenuItem, MenuItems } from '@headlessui/react';
import { openConfirm } from './ConfirmModal';
import { apiClient } from '@/utils/api';
import { isVideo, isAudio, encodeFilePathForUrl } from '@/utils/basic';
import { isVideo, isAudio, isText, encodeFilePathForUrl } from '@/utils/basic';
import AudioPlayer from './AudioPlayer';
import { TransformWrapper, TransformComponent } from 'react-zoom-pan-pinch';
import BoundingBoxOverlay, { parseBoundingBoxes } from './BoundingBoxOverlay';
@@ -199,6 +199,27 @@ export default function SampleImageViewer({
return imgPath;
}, [showingControlIdx, controlImages, imgPath]);
// text samples (LLM models): the file body is the sample
const [sampleText, setSampleText] = useState<string | null>(null);
useEffect(() => {
if (!displayedImgPath || !isText(displayedImgPath)) {
setSampleText(null);
return;
}
const controller = new AbortController();
fetch(`/api/img/${encodeFilePathForUrl(displayedImgPath)}`, { signal: controller.signal })
.then(r => (r.ok ? r.text() : Promise.reject(new Error(`HTTP ${r.status}`))))
.then(setSampleText)
.catch(err => {
if (err?.name !== 'AbortError') console.error('Sample text fetch failed:', err);
});
return () => controller.abort();
}, [displayedImgPath]);
// control thumbnails: audio has no image thumb, it gets its waveform art
const thumbSrc = (p: string) =>
isAudio(p) ? `/api/audio/art/${encodeURIComponent(p)}` : `/api/img/${encodeFilePathForUrl(p)}?thumb=1`;
// The sample's prompt is what generated it; if it's an Ideogram bbox-JSON we can
// overlay the boxes on the generated image. Only on the main image (not controls).
const boundingBoxes = useMemo(
@@ -311,7 +332,11 @@ export default function SampleImageViewer({
>
<div className="overflow-hidden flex items-center justify-center">
{displayedImgPath &&
(isAudio(displayedImgPath) ? (
(isText(displayedImgPath) ? (
<div className="w-[640px] max-w-full sm:max-w-[95vw] max-h-[82vh] overflow-y-auto p-6 text-sm text-gray-100 whitespace-pre-wrap break-words text-left">
{sampleText ?? 'Loading…'}
</div>
) : isAudio(displayedImgPath) ? (
<div className="w-[500px] h-[500px] max-w-full sm:max-w-[95vw] max-h-[82vh]">
<AudioPlayer
src={`/api/img/${encodeFilePathForUrl(displayedImgPath)}`}
@@ -370,19 +395,28 @@ export default function SampleImageViewer({
</div>
{controlImages.length > 0 && (
<div key={imgPath} className="flex space-x-2 mr-4">
{showingControlIdx !== null && (
<img
src={`/api/img/${encodeFilePathForUrl(imgPath!)}?thumb=1`}
alt="Main"
className="max-h-12 max-w-12 object-contain bg-black border-2 border-gray-700 hover:border-gray-500 rounded cursor-pointer"
onClick={() => setShowingControlIdx(null)}
title="Main image"
/>
)}
{showingControlIdx !== null &&
(isText(imgPath!) ? (
<div
className="h-12 w-12 flex items-center justify-center text-xs bg-black border-2 border-gray-700 hover:border-gray-500 rounded cursor-pointer"
onClick={() => setShowingControlIdx(null)}
title="Generated text"
>
Txt
</div>
) : (
<img
src={thumbSrc(imgPath!)}
alt="Main"
className="max-h-12 max-w-12 object-contain bg-black border-2 border-gray-700 hover:border-gray-500 rounded cursor-pointer"
onClick={() => setShowingControlIdx(null)}
title="Main image"
/>
))}
{controlImages.map((ci, idx) => (
<img
key={idx}
src={`/api/img/${encodeFilePathForUrl(ci)}?thumb=1`}
src={thumbSrc(ci)}
alt={`Control ${idx + 1}`}
className={`max-h-12 max-w-12 object-contain bg-black border-2 rounded cursor-pointer ${
showingControlIdx === idx ? 'border-blue-500' : 'border-gray-700 hover:border-gray-500'
@@ -429,7 +463,7 @@ export default function SampleImageViewer({
anchor="bottom end"
className="bg-gray-900 border border-gray-700 rounded shadow-lg w-48 px-2 py-2 mt-1 z-50"
>
{imgPath && isAudio(imgPath) && (
{imgPath && (isAudio(imgPath) || isText(imgPath)) && (
<MenuItem>
<a
className="cursor-pointer px-4 py-1 hover:bg-gray-800 rounded block"
ui/src/docs.tsx
+21 -15
View File
@@ -340,24 +340,30 @@ const docs: { [key: string]: ConfigDoc } = {
</>
),
},
'model.model_kwargs.instruction': {
title: 'LLM Prompt',
description: (
<>
The instruction the model is asked for every training item, as the user turn of the chat. The item's caption
file is the answer it learns to give. Use the same wording when running the trained LoRA, and it is also what
samples use when their own prompt is left blank.
</>
),
},
'model.model_kwargs.kv_cache': {
title: 'KV Cache',
description: (
<>
This will enable KV Cache for control images in a model that supports it. LoRAs trained with this on
need to also be inferenced with it, and vice versa. This does not speed up or slow down training, but on inference,
the control images only need to be processed once for the entire generation, vs being processed for every step.
This will enable KV Cache for control images in a model that supports it. LoRAs trained with this on need to
also be inferenced with it, and vice versa. This does not speed up or slow down training, but on inference, the
control images only need to be processed once for the entire generation, vs being processed for every step.
Which leads to a significant speedup on inference.
</>
),
},
'train.guidance_loss_target': {
title: 'Guidance Loss Target',
description: (
<>
For contrastive guidance loss, this is the target CGF to amplify predictions to.
</>
),
description: <>For contrastive guidance loss, this is the target CGF to amplify predictions to.</>,
},
'datasets.caption_dropout_rate': {
title: 'Caption Dropout Rate',
@@ -365,15 +371,15 @@ const docs: { [key: string]: ConfigDoc } = {
<>
Caption dropout rate is the probability that the caption for an image will be dropped (replaced with a blank
caption) for any given training step. For example, a value of 0.05 will drop the caption around 5% of the time.
Dropping captions helps the model learn the concept being trained without relying entirely on the caption,
and helps preserve the model&apos;s ability to generate without a prompt. If a trigger word is set, the trigger
word is still used when the caption is dropped, so the model still associates the dropped samples with your
trigger word. Regularization images, or images without a trigger word, drop to a fully blank caption.
Dropping captions helps the model learn the concept being trained without relying entirely on the caption, and
helps preserve the model&apos;s ability to generate without a prompt. If a trigger word is set, the trigger word
is still used when the caption is dropped, so the model still associates the dropped samples with your trigger
word. Regularization images, or images without a trigger word, drop to a fully blank caption.
<br />
<br />
Caption dropout also works when caching text embeddings. An additional embedding for the dropout caption
(blank, or the trigger word alone) is cached to disk alongside the normal one, and it is randomly swapped in
at train time at this rate.
Caption dropout also works when caching text embeddings. An additional embedding for the dropout caption (blank,
or the trigger word alone) is cached to disk alongside the normal one, and it is randomly swapped in at train
time at this rate.
</>
),
},
ui/src/helpers/defaultSamples.ts
+10
View File
@@ -985,6 +985,16 @@ const tagValue = (prompt: string, tag: string) => {
return m ? m[1].trim() : '';
};
// text-generating models: the prompt is the instruction, ctrl_img the media; output is a .txt
export const defaultQwen25OmniSampleConfig: SampleConfig = {
...defaultSampleConfig,
width: 512,
height: 512,
guidance_scale: 1,
sample_steps: 1,
samples: [{ prompt: 'Describe this in detail.' }, { prompt: 'Describe this in detail.' }],
};
export const defaultYue2SampleConfig: SampleConfig = {
...defaultAudioSampleConfig,
samples: defaultAudioSampleConfig.samples.map(s => ({