From 86b19589a01649a0fd50e2af106a93b213132069 Mon Sep 17 00:00:00 2001 From: Jaret Burkett Date: Tue, 16 Jun 2026 09:44:38 -0600 Subject: [PATCH] Update the Ideogram 4 prompt generation/parsing/ui to handle the updated format notes better. --- .../captioner/Ideogram4Captioner.py | 64 +--- .../prompts/ideogram4_caption_prompt.py | 32 +- .../captioner/prompts/ideogram4_prompt.py | 34 ++- .../prompts/ideogram4_upsample_prompt.py | 26 +- .../diffusion_models/ideogram4/ideogram4.py | 5 + toolkit/ideogram_caption.py | 282 ++++++++++++++++++ ui/src/components/IdeogramCaptionSidebar.tsx | 131 +++++++- ui/src/helpers/defaultSamples.ts | 20 +- ui_scripts/upsample_ideogram4_caption.py | 56 +--- version.py | 2 +- 10 files changed, 497 insertions(+), 155 deletions(-) create mode 100644 toolkit/ideogram_caption.py diff --git a/extensions_built_in/captioner/Ideogram4Captioner.py b/extensions_built_in/captioner/Ideogram4Captioner.py index 6e46048..dd06015 100644 --- a/extensions_built_in/captioner/Ideogram4Captioner.py +++ b/extensions_built_in/captioner/Ideogram4Captioner.py @@ -8,6 +8,7 @@ from PIL import Image from .Qwen3VLCaptioner import Qwen3VLCaptioner from .prompts.ideogram4_caption_prompt import ideogram4_caption_prompt +from toolkit.ideogram_caption import normalize_caption_dict import transformers import logging import warnings @@ -25,11 +26,6 @@ MIN_NEW_TOKENS = 3072 # generator was trained on, instead of ugly fractions like 1023:768. MAX_AR_DENOMINATOR = 16 -# color_palette caps: the model often ignores these, so we enforce them. -MAX_IMAGE_PALETTE = 16 # style_description.color_palette -MAX_ELEMENT_PALETTE = 5 # per-element color_palette -HEX_COLOR_RE = re.compile(r"^#(?:[0-9a-fA-F]{3}|[0-9a-fA-F]{6})$") - class Ideogram4Captioner(Qwen3VLCaptioner): def __init__(self, process_id: int, job, config: OrderedDict, **kwargs): @@ -109,65 +105,23 @@ class Ideogram4Captioner(Qwen3VLCaptioner): # stored order is [y1, x1, y2, x2] return [y1, x1, y2, x2] - def _sanitize_palette(self, palette, max_len): - """Keep unique, valid hex colors in order, capped to max_len. Returns the - cleaned list, or None if nothing valid remains (drop the key).""" - if not isinstance(palette, (list, tuple)): - return None - seen = set() - out = [] - for c in palette: - if not isinstance(c, str): - continue - c = c.strip() - if not HEX_COLOR_RE.match(c): - continue - key = c.lower() - if key in seen: - continue - seen.add(key) - out.append(c) - if len(out) >= max_len: - break - return out or None - def _normalize_caption(self, data: dict) -> dict: - """Validate/cleanup the parsed caption before storage: drop input-only - aspect_ratio, reorder bboxes to [y1,x1,y2,x2], and cap color palettes - (16 per image, 5 per element) since the model often exceeds them.""" - # aspect_ratio is input-only context, not part of the output. Drop it if - # the model echoed it anyway. - data.pop("aspect_ratio", None) - - style = data.get("style_description") - if isinstance(style, dict) and "color_palette" in style: - pal = self._sanitize_palette(style["color_palette"], MAX_IMAGE_PALETTE) - if pal is None: - style.pop("color_palette", None) - else: - style["color_palette"] = pal - + """Cleanup the parsed caption before storage. The model emits bboxes in + [x1,y1,x2,y2]; convert each to our stored [y1,x1,y2,x2] order, then hand off + to the shared normalizer for the rest: drop aspect_ratio, enforce the + photo/art_style branch and key order, canonicalize medium, and cap/uppercase + color palettes (16 per image, 5 per element).""" decon = data.get("compositional_deconstruction", {}) - elements = decon.get("elements", []) + elements = decon.get("elements", []) if isinstance(decon, dict) else [] if isinstance(elements, list): for el in elements: - if not isinstance(el, dict): - continue - if "bbox" in el: + if isinstance(el, dict) and "bbox" in el: cleaned = self._convert_bbox(el["bbox"]) if cleaned is None: el.pop("bbox", None) else: el["bbox"] = cleaned - if "color_palette" in el: - pal = self._sanitize_palette( - el["color_palette"], MAX_ELEMENT_PALETTE - ) - if pal is None: - el.pop("color_palette", None) - else: - el["color_palette"] = pal - return data + return normalize_caption_dict(data) def get_caption_for_file(self, file_path: str) -> Optional[str]: try: diff --git a/extensions_built_in/captioner/prompts/ideogram4_caption_prompt.py b/extensions_built_in/captioner/prompts/ideogram4_caption_prompt.py index 3d04742..4f42233 100644 --- a/extensions_built_in/captioner/prompts/ideogram4_caption_prompt.py +++ b/extensions_built_in/captioner/prompts/ideogram4_caption_prompt.py @@ -19,7 +19,7 @@ You are CAPTIONING a real image, not imagining one. Describe ONLY what is visibl ## OUTPUT CONTRACT — exactly three top-level keys, in this order: ```json -{"high_level_description":"...","style_description":{"aesthetics":"...","lighting":"...","photo":"...","medium":"...","color_palette":["#RRGGBB"]},"compositional_deconstruction":{"background":"...","elements":[ ... ]}} +{"high_level_description":"...","style_description":{ ...see STYLE DESCRIPTION... },"compositional_deconstruction":{"background":"...","elements":[ ... ]}} ``` - Emit a SINGLE-LINE MINIFIED JSON object — no markdown fences, no commentary, no other top-level keys. @@ -44,22 +44,34 @@ BAD (over-specifies): `A male soccer player captured mid-kick on a bright green ## STYLE DESCRIPTION — the `style_description` block (always required) -A nested object capturing the image's overall look, OBSERVED from the image (never invented). Exactly these five keys: -- `aesthetics` — the overall mood/aesthetic in a short phrase (`Cinematic, minimal, serene.` / `Bright, playful, high-energy.`). +A nested object capturing the image's overall look, OBSERVED from the image (never invented). It carries EXACTLY ONE render key — `photo` for photographs, `art_style` for everything else (illustration / 3D render / painting / graphic design) — NEVER both. The key order is strict and depends on the branch: + +- **Photograph** → keys in this order: `aesthetics`, `lighting`, `photo`, `medium`, `color_palette` + ```json + {"aesthetics":"...","lighting":"...","photo":"...","medium":"photograph","color_palette":["#RRGGBB"]} + ``` +- **Non-photo** (illustration / 3D / painting / graphic design) → keys in this order: `aesthetics`, `lighting`, `medium`, `art_style`, `color_palette` + ```json + {"aesthetics":"...","lighting":"...","medium":"illustration","art_style":"...","color_palette":["#RRGGBB"]} + ``` + +Field meanings: +- `aesthetics` — the overall mood/aesthetic in a short phrase (`cinematic, minimal, serene` / `bright, playful, high-energy`). - `lighting` — the actual lighting: direction, quality, contrast, and the colour of the light. Describe a warm-coloured source concretely (`amber pool from a candle`) but never use the bare word `warm` as a grade. -- `photo` — the medium-specific capture/render spec. Photograph → camera/film look, framing, grain, focus (`35mm film still, 16:9 framing, subtle grain, shallow depth of field`). Other media → the rendering technique (`flat vector, clean edges` / `octane 3D render, soft global illumination` / `loose watercolor on textured paper`). -- `medium` — one short phrase: `Photograph.` / `Illustration.` / `3D render.` / `Graphic design.` Read it from the image; do not impose a default. -- `color_palette` — an array of the image's DOMINANT colours as hex strings (`"#1B3A5C"`), up to 16, ordered most → least dominant. Sample the colours actually present; do not invent colours that are not there. +- `photo` (photographs ONLY) — the camera/film capture spec: framing, grain, focus (`35mm film still, 16:9 framing, subtle grain, shallow depth of field`). +- `art_style` (non-photo ONLY) — the rendering technique (`flat vector, clean edges` / `octane 3D render, soft global illumination` / `loose watercolor on textured paper`). +- `medium` — exactly one token: `photograph` / `illustration` / `3d_render` / `painting` / `graphic_design`. Read it from the image; do not impose a default. Photograph ⇒ use `photo`; any other ⇒ use `art_style`. +- `color_palette` — an array of the image's DOMINANT colours as UPPERCASE `#RRGGBB` hex strings (`"#1B3A5C"`), up to 16, ordered most → least dominant. Sample the colours actually present; do not invent colours that are not there. ALWAYS the last key. ## ELEMENTS — what they are, what they're not -Each element is one of: +Each element is one of (keys in EXACTLY this order): ``` -{"type":"obj","bbox":[x1,y1,x2,y2],"color_palette":["#RRGGBB"],"desc":"..."} -{"type":"text","bbox":[x1,y1,x2,y2],"color_palette":["#RRGGBB"],"text":"LINE ONE\\nLINE TWO","desc":"..."} +{"type":"obj","bbox":[x1,y1,x2,y2],"desc":"...","color_palette":["#RRGGBB"]} +{"type":"text","bbox":[x1,y1,x2,y2],"text":"LINE ONE\\nLINE TWO","desc":"...","color_palette":["#RRGGBB"]} ``` -`bbox` and `color_palette` are both OPTIONAL per-element. `bbox`: see BBOX section below. `color_palette`: up to 5 hex strings of that element's own dominant colours — include it when the element has distinctive colours worth pinning (a red jacket, a brand logo, coloured text), omit it for colour-neutral elements. +`bbox` and `color_palette` are both OPTIONAL per-element; when present they keep the order shown above (`color_palette` is always LAST). `bbox`: see BBOX section below. `color_palette`: up to 5 UPPERCASE `#RRGGBB` strings of that element's own dominant colours — include it when the element has distinctive colours worth pinning (a red jacket, a brand logo, coloured text), omit it for colour-neutral elements. ### SINGLE SUBJECT = SINGLE ELEMENT diff --git a/extensions_built_in/captioner/prompts/ideogram4_prompt.py b/extensions_built_in/captioner/prompts/ideogram4_prompt.py index 5d55040..5cba092 100644 --- a/extensions_built_in/captioner/prompts/ideogram4_prompt.py +++ b/extensions_built_in/captioner/prompts/ideogram4_prompt.py @@ -10,7 +10,7 @@ You convert a natural-language user idea into a structured JSON caption an image ## OUTPUT CONTRACT — exactly three top-level keys, in this order: ```json -{"high_level_description":"...","style_description":{"aesthetics":"...","lighting":"...","photo":"...","medium":"...","color_palette":["#RRGGBB"]},"compositional_deconstruction":{"background":"...","elements":[ ... ]}} +{"high_level_description":"...","style_description":{ ...see style_description... },"compositional_deconstruction":{"background":"...","elements":[ ... ]}} ``` - Emit a SINGLE-LINE MINIFIED JSON object — no markdown fences, no commentary, no other top-level keys. @@ -35,24 +35,29 @@ BAD (over-specifies): `A male soccer player captured mid-kick on a bright green ### `style_description` — the global look block (always required) -A nested object with exactly these five keys: -- `aesthetics` — overall mood/aesthetic in a short phrase (`Cinematic, minimal, serene.`). +A nested object carrying EXACTLY ONE render key — `photo` for photographs, `art_style` for everything else — NEVER both. Key order is strict and branch-dependent: + +- **Photograph** → `aesthetics`, `lighting`, `photo`, `medium`, `color_palette` +- **Non-photo** (illustration / 3D / painting / graphic design) → `aesthetics`, `lighting`, `medium`, `art_style`, `color_palette` + +- `aesthetics` — overall mood/aesthetic in a short phrase (`cinematic, minimal, serene`). - `lighting` — direction, quality, contrast, and colour of the light. Describe a warm-coloured source concretely (`amber sun low at the horizon`); never use the bare word `warm` as a grade. -- `photo` — the medium-specific capture/render spec. Photograph → camera/film look, framing, grain, focus (`35mm motion-picture film still, 16:9 framing, subtle grain`). Other media → the rendering technique (`flat vector, clean edges`; `octane 3D render`; `loose watercolor on textured paper`). -- `medium` — one short phrase: `Photograph.` / `Illustration.` / `3D render.` / `Graphic design.` -- `color_palette` — an array of the dominant colours as hex strings (`"#1B3A5C"`), up to 16, ordered most → least dominant. This conditions the image's colours directly, so commit to the actual hexes you intend. +- `photo` (photographs ONLY) — the camera/film capture spec: framing, grain, focus (`35mm motion-picture film still, 16:9 framing, subtle grain`). +- `art_style` (non-photo ONLY) — the rendering technique (`flat vector, clean edges`; `octane 3D render`; `loose watercolor on textured paper`). +- `medium` — exactly one token: `photograph` / `illustration` / `3d_render` / `painting` / `graphic_design`. Photograph ⇒ use `photo`; any other ⇒ use `art_style`. +- `color_palette` — an array of the dominant colours as UPPERCASE `#RRGGBB` hex strings (`"#1B3A5C"`), up to 16, ordered most → least dominant. This conditions the image's colours directly, so commit to the actual hexes you intend. ALWAYS the last key. Name a recognized style ONCE here (see PLANNING → Style commitment); do not append invented technique detail on top of a well-known style name. ## ELEMENTS — what they are, what they're not -Each element is one of: +Each element is one of (keys in EXACTLY this order): ``` -{"type":"obj","bbox":[y1,x1,y2,x2],"color_palette":["#RRGGBB"],"desc":"..."} -{"type":"text","bbox":[y1,x1,y2,x2],"color_palette":["#RRGGBB"],"text":"LINE ONE\nLINE TWO","desc":"..."} +{"type":"obj","bbox":[y1,x1,y2,x2],"desc":"...","color_palette":["#RRGGBB"]} +{"type":"text","bbox":[y1,x1,y2,x2],"text":"LINE ONE\nLINE TWO","desc":"...","color_palette":["#RRGGBB"]} ``` -`bbox` and `color_palette` are both OPTIONAL per-element. `bbox`: see BBOX section below. `color_palette`: up to 5 hex strings steering that element's own dominant colours — include it when the element has a distinctive colour (a red jacket, a brand logo, coloured text), omit it otherwise. +`bbox` and `color_palette` are both OPTIONAL per-element; when present they keep the order shown (`color_palette` is always LAST). `bbox`: see BBOX section below. `color_palette`: up to 5 UPPERCASE `#RRGGBB` strings steering that element's own dominant colours — include it when the element has a distinctive colour (a red jacket, a brand logo, coloured text), omit it otherwise. ### SINGLE SUBJECT = SINGLE ELEMENT @@ -224,13 +229,14 @@ The "dense unenumerable group" exception (crowd of thousands, field of wildflowe ### 1. Pick a medium -`photograph | illustration | 3D render | graphic design` — applies as natural-language framing inside HLD/background, NOT as a structured slot. +`photograph | illustration | 3d_render | painting | graphic_design` — this is the `medium` token (photograph ⇒ `photo`, all others ⇒ `art_style`), and it also frames HLD/background prose naturally. Decision: **DESIGNED artifact vs CAPTURED / DRAWN / RENDERED moment.** -- **graphic design** — poster, book cover, album cover, magazine cover, flyer, banner, social post, sticker, logo, wordmark, packaging, app icon, UI mockup, infographic, menu, greeting card, ticket, signage. If a human designer would sit at a desk to make it. +- **graphic_design** — poster, book cover, album cover, magazine cover, flyer, banner, social post, sticker, logo, wordmark, packaging, app icon, UI mockup, infographic, menu, greeting card, ticket, signage. If a human designer would sit at a desk to make it. - **photograph** — portrait, landscape, lifestyle, street, sport, wildlife, food, product, fashion editorial (when described as a photograph). Default for ambiguous everyday scenes. -- **illustration** — cartoon, anime, manga, comic, watercolor, oil painting, ink, vector, pixel art, children's book illustration, named studios (Ghibli, KyoAni, Pixar 2D). -- **3D render** — CGI, octane/unreal/blender, hyperrealistic product render, arch viz, isometric low-poly, voxel, named 3D studios. +- **illustration** — cartoon, anime, manga, comic, ink, vector, pixel art, children's book illustration, named studios (Ghibli, KyoAni, Pixar 2D). +- **painting** — watercolor, oil, gouache, acrylic, traditional painterly work. +- **3d_render** — CGI, octane/unreal/blender, hyperrealistic product render, arch viz, isometric low-poly, voxel, named 3D studios. Silent / ambiguous → photograph (default). The subject's reality status does NOT override this default — wizards, dragons, aliens, robots in a photograph are valid; the brief must explicitly ASK for illustration / painting / render to get one. diff --git a/extensions_built_in/captioner/prompts/ideogram4_upsample_prompt.py b/extensions_built_in/captioner/prompts/ideogram4_upsample_prompt.py index 31a0176..d1b0e67 100644 --- a/extensions_built_in/captioner/prompts/ideogram4_upsample_prompt.py +++ b/extensions_built_in/captioner/prompts/ideogram4_upsample_prompt.py @@ -17,7 +17,7 @@ You convert a user prompt into a structured JSON caption an image renderer can c ## OUTPUT CONTRACT — exactly three top-level keys, in this order: ```json -{"high_level_description":"...","style_description":{"aesthetics":"...","lighting":"...","photo":"...","medium":"...","color_palette":["#RRGGBB"]},"compositional_deconstruction":{"background":"...","elements":[ ... ]}} +{"high_level_description":"...","style_description":{ ...see STYLE DESCRIPTION... },"compositional_deconstruction":{"background":"...","elements":[ ... ]}} ``` - Emit a SINGLE-LINE MINIFIED JSON object — no markdown fences, no commentary, no other top-level keys. @@ -34,23 +34,29 @@ One short sentence, reads like a natural prompt, starts with the subject — no ## STYLE DESCRIPTION — the `style_description` block (always required) -A nested object with exactly these five keys, filled FROM the prompt: +A nested object, filled FROM the prompt. It carries EXACTLY ONE render key — `photo` for photographs, `art_style` for everything else — NEVER both. Key order is strict and branch-dependent: + +- **Photograph** → `aesthetics`, `lighting`, `photo`, `medium`, `color_palette` +- **Non-photo** (illustration / 3D / painting / graphic design) → `aesthetics`, `lighting`, `medium`, `art_style`, `color_palette` + +Fields: - `aesthetics` — the overall mood/aesthetic in a short phrase. - `lighting` — the lighting (direction, quality, colour). Describe a warm-coloured source concretely; never use the bare word `warm` as a grade. -- `photo` — the medium-specific capture/render spec (photograph → camera/film look, framing, grain, focus; other media → the rendering technique). -- `medium` — one short phrase: `Photograph.` / `Illustration.` / `3D render.` / `Graphic design.` -- `color_palette` — an array of dominant colours as hex strings (`"#1B3A5C"`), up to 16, ordered most → least dominant. +- `photo` (photographs ONLY) — the camera/film capture spec (framing, grain, focus). +- `art_style` (non-photo ONLY) — the rendering technique (`flat vector, clean edges`; `octane 3D render`; `loose watercolor`). +- `medium` — exactly one token: `photograph` / `illustration` / `3d_render` / `painting` / `graphic_design`. Photograph ⇒ use `photo`; any other ⇒ use `art_style`. +- `color_palette` — an array of dominant colours as UPPERCASE `#RRGGBB` strings (`"#1B3A5C"`), up to 16, ordered most → least dominant. ALWAYS the last key. -Respect FIDELITY: if the prompt NAMES a style, medium, artist, or look, put it in these fields BY NAME (e.g. `medium`/`photo`/`aesthetics`) and do NOT invent its characteristics. Pull lighting and colours from what the prompt states. In faithful mode, only commit to a value the prompt implies, keeping the rest minimal; in creative mode you may infer fitting style values — but never elaborate a named style and never override what the user gave. +Respect FIDELITY: if the prompt NAMES a style, medium, artist, or look, put it in these fields BY NAME (e.g. `medium`/`art_style`/`aesthetics`) and do NOT invent its characteristics. Pull lighting and colours from what the prompt states. In faithful mode, only commit to a value the prompt implies, keeping the rest minimal; in creative mode you may infer fitting style values — but never elaborate a named style and never override what the user gave. ## ELEMENTS -Each element is one of: +Each element is one of (keys in EXACTLY this order): ``` -{"type":"obj","bbox":[y1,x1,y2,x2],"color_palette":["#RRGGBB"],"desc":"..."} -{"type":"text","bbox":[y1,x1,y2,x2],"color_palette":["#RRGGBB"],"text":"LINE ONE\nLINE TWO","desc":"..."} +{"type":"obj","bbox":[y1,x1,y2,x2],"desc":"...","color_palette":["#RRGGBB"]} +{"type":"text","bbox":[y1,x1,y2,x2],"text":"LINE ONE\nLINE TWO","desc":"...","color_palette":["#RRGGBB"]} ``` -`bbox` and `color_palette` are both OPTIONAL per element. `bbox`: see BBOX. `color_palette`: up to 5 hex strings of that element's dominant colours — include it when the prompt gives the element a distinctive colour (a red jacket, coloured text), otherwise omit. +`bbox` and `color_palette` are both OPTIONAL per element; when present they keep the order shown (`color_palette` is always LAST). `bbox`: see BBOX. `color_palette`: up to 5 UPPERCASE `#RRGGBB` strings of that element's dominant colours — include it when the prompt gives the element a distinctive colour (a red jacket, coloured text), otherwise omit. - **One coherent subject = ONE element.** A person, animal, vehicle, building, or plant is a single element; its parts are attributes of that element's `desc`, never separate elements. Multiple distinct subjects = multiple elements (one each). - **`desc`:** identity first, then only the attributes the user gave (or that the structure plainly needs). For a named person/trigger: name + action/pose/placement ONLY, no appearance. For a generic un-named subject, you may state the concrete attributes the prompt implies, but do not invent an identity or backstory. diff --git a/extensions_built_in/diffusion_models/ideogram4/ideogram4.py b/extensions_built_in/diffusion_models/ideogram4/ideogram4.py index 810cd99..ea17b4c 100644 --- a/extensions_built_in/diffusion_models/ideogram4/ideogram4.py +++ b/extensions_built_in/diffusion_models/ideogram4/ideogram4.py @@ -10,6 +10,7 @@ from toolkit.models.base_model import BaseModel from toolkit.basic import flush from toolkit.print import print_acc from toolkit.advanced_prompt_embeds import AdvancedPromptEmbeds +from toolkit.ideogram_caption import digest_caption_string from toolkit.samplers.custom_flowmatch_sampler import ( CustomFlowMatchEulerDiscreteScheduler, ) @@ -419,6 +420,10 @@ class Ideogram4Model(BaseModel): # length -- important for the long structured (JSON) captions. features_list = [] for p in prompt: + # Digest the prompt: migrate any old-format Ideogram caption into the + # current schema and serialize it compact (the form the renderer wants). + # Plain-text prompts pass straight through unchanged. + p = digest_caption_string(p) messages = [{"role": "user", "content": [{"type": "text", "text": p}]}] text = self.tokenizer.apply_chat_template( messages, add_generation_prompt=True, tokenize=False diff --git a/toolkit/ideogram_caption.py b/toolkit/ideogram_caption.py new file mode 100644 index 0000000..bfbc1c8 --- /dev/null +++ b/toolkit/ideogram_caption.py @@ -0,0 +1,282 @@ +"""Shared helpers for Ideogram-4 structured JSON captions. + +This is the single source of truth for the caption schema so the captioner, the +prompt upsampler, the dataloader, and the model encoder all agree. It encodes the +official Ideogram-4 rules and, crucially, MIGRATES the old caption format we used +before those rules were published into the new one ("digest" old, emit new). + +Official schema (summary): +- three top-level keys: high_level_description (optional), style_description + (optional), compositional_deconstruction (required). +- style_description holds EXACTLY ONE of `photo` (photographs) or `art_style` + (illustration/painting/3D/graphic design), never both. Key order is strict and + branch-dependent: + photo branch: aesthetics, lighting, photo, medium, color_palette + non-photo branch: aesthetics, lighting, medium, art_style, color_palette +- medium is one of: photograph, illustration, 3d_render, painting, graphic_design +- color_palette: UPPERCASE #RRGGBB only, up to 16 per image / 5 per element. +- elements, strict key order: + obj: type, bbox, desc, color_palette + text: type, bbox, text, desc, color_palette + bbox is optional, normalized 0-1000, [y_min, x_min, y_max, x_max], top-left. +- serialize compact: separators=(",", ":"), ensure_ascii=False (no \\uXXXX). + +The OLD format we previously emitted differed by: always using `photo` (even for +non-photo media), putting `color_palette` before `desc`/`text`, title-cased medium +with a trailing period ("Illustration."), and lowercase / 3-digit hex. Every +function here accepts the old shape and returns the new one. +""" + +import json +import re +from collections import OrderedDict + +MAX_IMAGE_PALETTE = 16 # style_description.color_palette +MAX_ELEMENT_PALETTE = 5 # per-element color_palette + +# Canonical medium tokens (official set). +MEDIUM_OPTIONS = [ + "photograph", + "illustration", + "3d_render", + "painting", + "graphic_design", +] + +# Map common variants (including our old "Title." style) to the canonical token. +# Anything not listed is treated as a custom medium and preserved verbatim. +_MEDIUM_ALIASES = { + "photograph": "photograph", + "photo": "photograph", + "illustration": "illustration", + "3d render": "3d_render", + "3d_render": "3d_render", + "3d-render": "3d_render", + "3drender": "3d_render", + "render": "3d_render", + "3d": "3d_render", + "painting": "painting", + "graphic design": "graphic_design", + "graphic_design": "graphic_design", + "graphic-design": "graphic_design", + "graphic": "graphic_design", +} + +_HEX6_RE = re.compile(r"^#[0-9a-fA-F]{6}$") +_HEX3_RE = re.compile(r"^#[0-9a-fA-F]{3}$") + + +def canon_medium(medium): + """Canonicalize a medium string to an official token when recognized, + otherwise return it stripped (custom mediums are allowed, preserved as-is).""" + if not isinstance(medium, str): + return medium + key = medium.strip().rstrip(".").strip().lower() + if key in _MEDIUM_ALIASES: + return _MEDIUM_ALIASES[key] + return medium.strip() + + +def is_photo_medium(medium): + """True for the photograph branch (uses `photo`), False for the art_style branch.""" + return canon_medium(medium) == "photograph" + + +def normalize_hex(color): + """Return an UPPERCASE #RRGGBB string, expanding #RGB -> #RRGGBB. None if invalid.""" + if not isinstance(color, str): + return None + s = color.strip() + if _HEX6_RE.match(s): + return "#" + s[1:].upper() + if _HEX3_RE.match(s): + return "#" + "".join(ch * 2 for ch in s[1:]).upper() + return None + + +def sanitize_palette(palette, max_len): + """Keep unique, valid, UPPERCASE hex colors in order, capped to max_len. + Returns the cleaned list, or None if nothing valid remains (drop the key).""" + if not isinstance(palette, (list, tuple)): + return None + seen = set() + out = [] + for c in palette: + h = normalize_hex(c) + if h is None or h in seen: + continue + seen.add(h) + out.append(h) + if len(out) >= max_len: + break + return out or None + + +def normalize_style(style): + """Reorder/clean style_description into the correct branch (photo vs art_style) + with the strict key order, canonical medium, and uppercase palette. Accepts the + old shape (always `photo`) and migrates it based on the medium.""" + if not isinstance(style, dict): + return style + + raw_medium = style.get("medium") + medium = canon_medium(raw_medium) if raw_medium is not None else None + has_photo = bool(style.get("photo")) + has_art = bool(style.get("art_style")) + + # Decide the branch. A recognized medium is authoritative; otherwise infer from + # whichever render key the (old) data already had, defaulting to photo. + if medium in MEDIUM_OPTIONS: + photo_branch = medium == "photograph" + elif has_art and not has_photo: + photo_branch = False + else: + photo_branch = True + + photo_val = style.get("photo") if has_photo else None + art_val = style.get("art_style") if has_art else None + + out = OrderedDict() + if "aesthetics" in style: + out["aesthetics"] = style["aesthetics"] + if "lighting" in style: + out["lighting"] = style["lighting"] + + if photo_branch: + # aesthetics, lighting, photo, medium, color_palette + val = photo_val if photo_val is not None else art_val + if val is not None: + out["photo"] = val + if medium is not None: + out["medium"] = medium + else: + # aesthetics, lighting, medium, art_style, color_palette + if medium is not None: + out["medium"] = medium + val = art_val if art_val is not None else photo_val + if val is not None: + out["art_style"] = val + + pal = sanitize_palette(style.get("color_palette"), MAX_IMAGE_PALETTE) + if pal is not None: + out["color_palette"] = pal + + # Preserve any unexpected extra keys at the end rather than dropping them. + for k, v in style.items(): + if k not in ( + "aesthetics", + "lighting", + "photo", + "art_style", + "medium", + "color_palette", + ): + out[k] = v + return out + + +def normalize_element(el): + """Reorder an element's keys to the strict schema order and uppercase its + palette. obj: type, bbox, desc, color_palette. text: type, bbox, text, desc, + color_palette. bbox is kept verbatim (already [y1,x1,y2,x2] in stored form).""" + if not isinstance(el, dict): + return el + etype = el.get("type", "obj") + out = OrderedDict() + out["type"] = etype + if el.get("bbox") is not None: + out["bbox"] = el["bbox"] + if etype == "text": + if "text" in el: + out["text"] = el["text"] + if "desc" in el: + out["desc"] = el["desc"] + else: + if "desc" in el: + out["desc"] = el["desc"] + pal = sanitize_palette(el.get("color_palette"), MAX_ELEMENT_PALETTE) + if pal is not None: + out["color_palette"] = pal + # Preserve any extras (e.g. future keys) at the end. + for k, v in el.items(): + if k not in out and k != "color_palette": + out[k] = v + return out + + +def normalize_caption_dict(data): + """Normalize a parsed caption dict in place-ish: drop input-only aspect_ratio, + enforce top-level key order, normalize style (photo/art_style branch) and every + element. Returns a new OrderedDict. Accepts old-format captions and emits new.""" + if not isinstance(data, dict): + return data + data.pop("aspect_ratio", None) # input-only context, never part of output + + out = OrderedDict() + if "high_level_description" in data: + out["high_level_description"] = data["high_level_description"] + if "style_description" in data: + out["style_description"] = normalize_style(data["style_description"]) + + decon = data.get("compositional_deconstruction") + if isinstance(decon, dict): + nd = OrderedDict() + if "background" in decon: + nd["background"] = decon["background"] + els = decon.get("elements") + if isinstance(els, list): + nd["elements"] = [normalize_element(e) for e in els] + for k, v in decon.items(): + if k not in ("background", "elements"): + nd[k] = v + out["compositional_deconstruction"] = nd + elif decon is not None: + out["compositional_deconstruction"] = decon + + for k, v in data.items(): + if k not in ( + "high_level_description", + "style_description", + "compositional_deconstruction", + "aspect_ratio", + ): + out[k] = v + return out + + +def is_ideogram_caption_str(text): + """True if text parses as a JSON object with a compositional_deconstruction block.""" + t = (text or "").strip() + if not t.startswith("{"): + return False + try: + d = json.loads(t) + except Exception: + return False + return isinstance(d, dict) and isinstance( + d.get("compositional_deconstruction"), dict + ) + + +def to_model_string(data): + """Serialize a caption dict to the compact, model-ready string the renderer wants.""" + return json.dumps(data, ensure_ascii=False, separators=(",", ":")) + + +def digest_caption_string(text): + """Parse, normalize (migrating old format), and return the compact model-ready + string. Returns the input unchanged if it is not an Ideogram structured caption + (plain-text captions pass straight through).""" + t = (text or "").strip() + if not t.startswith("{"): + return text + try: + data = json.loads(t, object_pairs_hook=OrderedDict) + except Exception: + return text + if not ( + isinstance(data, dict) + and isinstance(data.get("compositional_deconstruction"), dict) + ): + return text + return to_model_string(normalize_caption_dict(data)) diff --git a/ui/src/components/IdeogramCaptionSidebar.tsx b/ui/src/components/IdeogramCaptionSidebar.tsx index 292d948..896fb8b 100644 --- a/ui/src/components/IdeogramCaptionSidebar.tsx +++ b/ui/src/components/IdeogramCaptionSidebar.tsx @@ -18,10 +18,47 @@ export function isIdeogramCaption(text: string): boolean { } } -// Normalize an arbitrary color string to a #rrggbb value usable by . +// Official medium tokens. `photograph` uses the `photo` style key; every other +// medium uses `art_style`. Users may also type a custom medium. +const MEDIUM_OPTIONS = ['photograph', 'illustration', '3d_render', 'painting', 'graphic_design']; + +// Map old/variant medium spellings (e.g. our old "Illustration." / "3D render.") +// to a canonical token so the dropdown can match them; unknown values stay custom. +function canonMedium(m: string): string { + const key = (m || '').trim().replace(/\.+$/, '').trim().toLowerCase(); + const aliases: Record = { + photograph: 'photograph', + photo: 'photograph', + illustration: 'illustration', + '3d render': '3d_render', + '3d_render': '3d_render', + '3d-render': '3d_render', + '3drender': '3d_render', + render: '3d_render', + '3d': '3d_render', + painting: 'painting', + 'graphic design': 'graphic_design', + graphic_design: 'graphic_design', + 'graphic-design': 'graphic_design', + graphic: 'graphic_design', + }; + return aliases[key] ?? (m || '').trim(); +} + +// Photograph branch uses `photo`; everything else uses `art_style`. An unknown +// (empty/custom) medium defaults to the photo branch. +function isPhotoMedium(m: string): boolean { + const c = canonMedium(m); + if (c === 'photograph') return true; + if (MEDIUM_OPTIONS.includes(c)) return false; + return true; +} + +// Normalize a color to an UPPERCASE #RRGGBB value (expands #RGB). Returns +// '#000000' for anything unparseable (used to feed ). function toHex6(c: string): string { const s = (c || '').trim(); - if (/^#[0-9a-fA-F]{6}$/.test(s)) return s; + if (/^#[0-9a-fA-F]{6}$/.test(s)) return '#' + s.slice(1).toUpperCase(); if (/^#[0-9a-fA-F]{3}$/.test(s)) { return ( '#' + @@ -30,6 +67,7 @@ function toHex6(c: string): string { .split('') .map(ch => ch + ch) .join('') + .toUpperCase() ); } return '#000000'; @@ -99,6 +137,49 @@ function TextAreaField({ ); } +// Medium picker: a dropdown of the official tokens plus a "Custom…" escape hatch +// that reveals a free-text input. Recognizes old/variant spellings via canonMedium. +function MediumField({ value, onChange }: { value: string; onChange: (v: string) => void }) { + const canon = canonMedium(value); + const known = MEDIUM_OPTIONS.includes(canon); + const [custom, setCustom] = useState(!known && (value || '').trim() !== ''); + const showCustom = custom || (!known && (value || '').trim() !== ''); + return ( + + ); +} + function ColorPalette({ colors, max, onChange }: { colors: string[]; max: number; onChange: (c: string[]) => void }) { const setAt = (i: number, v: string) => onChange(colors.map((c, idx) => (idx === i ? v : c))); const removeAt = (i: number) => onChange(colors.filter((_, idx) => idx !== i)); @@ -109,7 +190,7 @@ function ColorPalette({ colors, max, onChange }: { colors: string[]; max: number setAt(i, e.target.value)} + onChange={e => setAt(i, toHex6(e.target.value))} className="w-5 h-5 rounded cursor-pointer bg-transparent border-0 p-0" title="Pick color" /> @@ -218,6 +299,40 @@ export default function IdeogramCaptionSidebar({ d.style_description = sd; }); + // Setting the medium may flip the branch (photograph ↔ non-photo). When it does, + // migrate the render text from photo↔art_style so we never keep both keys. + const setMedium = (value: string) => + update(d => { + const sd = { ...(d.style_description || {}) }; + const wasPhoto = isPhotoMedium(sd.medium ?? ''); + const nowPhoto = isPhotoMedium(value); + sd.medium = value; + if (wasPhoto !== nowPhoto) { + const val = sd.photo ?? sd.art_style ?? ''; + delete sd.photo; + delete sd.art_style; + if (nowPhoto) sd.photo = val; + else sd.art_style = val; + } + d.style_description = sd; + }); + + // The single render field writes to `photo` or `art_style` per the current branch. + const photoBranch = isPhotoMedium(style.medium ?? ''); + const renderValue = (photoBranch ? style.photo : style.art_style) ?? style.photo ?? style.art_style ?? ''; + const setRender = (value: string) => + update(d => { + const sd = { ...(d.style_description || {}) }; + if (isPhotoMedium(sd.medium ?? '')) { + sd.photo = value; + delete sd.art_style; + } else { + sd.art_style = value; + delete sd.photo; + } + d.style_description = sd; + }); + const setElement = (i: number, mutator: (el: any) => void) => update(d => { const els = d?.compositional_deconstruction?.elements; @@ -279,12 +394,12 @@ export default function IdeogramCaptionSidebar({
setStyle('aesthetics', v)} /> setStyle('lighting', v)} /> - setStyle('photo', v)} /> + setStyle('medium', v)} - placeholder="Photograph." + label={photoBranch ? 'Photo (camera / film)' : 'Art style (rendering technique)'} + value={renderValue} + onChange={setRender} + placeholder={photoBranch ? '35mm film still, shallow depth of field' : 'flat vector, clean edges'} />
Color palette (max 16) diff --git a/ui/src/helpers/defaultSamples.ts b/ui/src/helpers/defaultSamples.ts index ac197b2..9b2e9f4 100644 --- a/ui/src/helpers/defaultSamples.ts +++ b/ui/src/helpers/defaultSamples.ts @@ -417,7 +417,7 @@ export const defaultIdeogramSamplesConfig: SampleConfig = { "aesthetics": "Cinematic, tense, candid realism.", "lighting": "Overcast afternoon daylight, soft and low-contrast, cool-neutral white balance.", "photo": "35mm film still, subtle grain, natural depth of field.", - "medium": "Photograph.", + "medium": "photograph", "color_palette": ["#8C9B82", "#B7402A", "#5A5F57", "#9AA7AE", "#D98A3D"] }, "compositional_deconstruction": { @@ -466,7 +466,7 @@ export const defaultIdeogramSamplesConfig: SampleConfig = { "aesthetics": "Cozy, relaxed, intimate.", "lighting": "Soft diffused window daylight, cool-neutral white balance, low contrast.", "photo": "35mm film still, shallow depth of field, subtle grain.", - "medium": "Photograph.", + "medium": "photograph", "color_palette": ["#9A5A3E", "#E0D2BA", "#7C7872", "#B07C45", "#33312D"] }, "compositional_deconstruction": { @@ -536,7 +536,7 @@ export const defaultIdeogramSamplesConfig: SampleConfig = { "aesthetics": "High-energy, surreal, neon nightlife.", "lighting": "Dim club lighting with magenta and cyan washes and crisscrossing green and magenta laser beams cutting through haze.", "photo": "Fish-eye lens with strong barrel distortion, deep shadow contrast.", - "medium": "Photograph.", + "medium": "photograph", "color_palette": ["#0B0B12", "#D81E8F", "#1FB6C9", "#37C46A", "#6A4A2E"] }, "compositional_deconstruction": { @@ -596,7 +596,7 @@ export const defaultIdeogramSamplesConfig: SampleConfig = { "aesthetics": "Bright, playful, candid.", "lighting": "Bright overcast daylight, soft and shadowless, cool-neutral white balance.", "photo": "35mm film still, natural depth of field, subtle grain.", - "medium": "Photograph.", + "medium": "photograph", "color_palette": ["#C9B68C", "#2E6B7A", "#9FB7BE", "#1B3A5C", "#E7E2D6"] }, "compositional_deconstruction": { @@ -646,7 +646,7 @@ export const defaultIdeogramSamplesConfig: SampleConfig = { "aesthetics": "Serene, rugged, wintry.", "lighting": "Pale overcast winter daylight, even and shadowless, cool-neutral white balance.", "photo": "35mm film still, subtle grain, soft natural focus.", - "medium": "Photograph.", + "medium": "photograph", "color_palette": ["#E8EDF0", "#6B4A30", "#3C5240", "#9AA6AD", "#C8A877"] }, "compositional_deconstruction": { @@ -705,7 +705,7 @@ export const defaultIdeogramSamplesConfig: SampleConfig = { "aesthetics": "Gritty, energetic, high-contrast.", "lighting": "Dark stage lit by green and magenta laser beams through haze, deep shadow contrast, cool-neutral white balance.", "photo": "35mm concert photograph, subtle grain, deep contrast.", - "medium": "Photograph.", + "medium": "photograph", "color_palette": ["#0C0C10", "#37C46A", "#D81E8F", "#C9C9C9", "#5A4633"] }, "compositional_deconstruction": { @@ -775,7 +775,7 @@ export const defaultIdeogramSamplesConfig: SampleConfig = { "aesthetics": "Rustic, focused, artisanal.", "lighting": "Diffused overcast daylight from a high window, cool-neutral white balance, low contrast.", "photo": "35mm film still, subtle grain, natural depth of field.", - "medium": "Photograph.", + "medium": "photograph", "color_palette": ["#8A5A3C", "#6E4327", "#9A9488", "#4A5340", "#C7B299"] }, "compositional_deconstruction": { @@ -845,7 +845,7 @@ export const defaultIdeogramSamplesConfig: SampleConfig = { "aesthetics": "Clean, minimal, editorial.", "lighting": "Even diffused studio softbox lighting, neutral white balance, shadowless.", "photo": "Studio fashion photograph, sharp focus, seamless white cyclorama.", - "medium": "Photograph.", + "medium": "photograph", "color_palette": ["#F2F2F0", "#9A9CA0", "#2A2F3C", "#5B5E66", "#D8D8D6"] }, "compositional_deconstruction": { @@ -874,7 +874,7 @@ export const defaultIdeogramSamplesConfig: SampleConfig = { "aesthetics": "Plain, candid, documentary.", "lighting": "Overcast daylight, soft and even, cool-neutral white balance.", "photo": "35mm film still, eye-level, subtle grain.", - "medium": "Photograph.", + "medium": "photograph", "color_palette": ["#9AA0A4", "#7C4A38", "#3A3D44", "#1E3A66", "#E8E6E0"] }, "compositional_deconstruction": { @@ -924,7 +924,7 @@ export const defaultIdeogramSamplesConfig: SampleConfig = { "aesthetics": "Rugged, cinematic, post-apocalyptic.", "lighting": "Pale dust-choked daylight softened by airborne grit, cool-neutral white balance, low contrast.", "photo": "35mm film still, subtle grain, hazy distance.", - "medium": "Photograph.", + "medium": "photograph", "color_palette": ["#C2A878", "#6B4A2E", "#3A352E", "#9A8A6C", "#B5562A"] }, "compositional_deconstruction": { diff --git a/ui_scripts/upsample_ideogram4_caption.py b/ui_scripts/upsample_ideogram4_caption.py index 7f60188..1ccc850 100644 --- a/ui_scripts/upsample_ideogram4_caption.py +++ b/ui_scripts/upsample_ideogram4_caption.py @@ -21,6 +21,8 @@ REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) if REPO_ROOT not in sys.path: sys.path.insert(0, REPO_ROOT) +from toolkit.ideogram_caption import normalize_caption_dict + # The generation prompt lives here. It's a `name = """"""` file, but the # content intentionally contains literal `\uNNNN` and `\n` sequences that are not # valid Python escapes, so it cannot be imported -- we read the triple-quoted @@ -127,62 +129,22 @@ def sanitize_bbox(bbox): return [y1, x1, y2, x2] -HEX_COLOR_RE = re.compile(r"^#(?:[0-9a-fA-F]{3}|[0-9a-fA-F]{6})$") - - -def sanitize_palette(palette, max_len): - """Keep unique, valid hex colors in order, capped to max_len. Returns the - cleaned list, or None if nothing valid remains (drop the key).""" - if not isinstance(palette, (list, tuple)): - return None - seen = set() - out = [] - for c in palette: - if not isinstance(c, str): - continue - c = c.strip() - if not HEX_COLOR_RE.match(c): - continue - key = c.lower() - if key in seen: - continue - seen.add(key) - out.append(c) - if len(out) >= max_len: - break - return out or None - - def sanitize_caption(data: dict) -> dict: - """Light cleanup: drop any aspect_ratio key (input-only context, not output), - clean each bbox, and cap color palettes (16 per image, 5 per element).""" - data.pop("aspect_ratio", None) - style = data.get("style_description") - if isinstance(style, dict) and "color_palette" in style: - pal = sanitize_palette(style["color_palette"], 16) - if pal is None: - style.pop("color_palette", None) - else: - style["color_palette"] = pal + """Clamp each bbox to valid 0-1000 [y1,x1,y2,x2], then hand off to the shared + normalizer for the rest: drop aspect_ratio, enforce the photo/art_style branch + and key order, canonicalize medium, and cap/uppercase color palettes (16 per + image, 5 per element).""" decon = data.get("compositional_deconstruction", {}) - elements = decon.get("elements", []) + elements = decon.get("elements", []) if isinstance(decon, dict) else [] if isinstance(elements, list): for el in elements: - if not isinstance(el, dict): - continue - if "bbox" in el: + if isinstance(el, dict) and "bbox" in el: cleaned = sanitize_bbox(el["bbox"]) if cleaned is None: el.pop("bbox", None) else: el["bbox"] = cleaned - if "color_palette" in el: - pal = sanitize_palette(el["color_palette"], 5) - if pal is None: - el.pop("color_palette", None) - else: - el["color_palette"] = pal - return data + return normalize_caption_dict(data) def upsample_one( diff --git a/version.py b/version.py index 4c65788..46ee4ed 100644 --- a/version.py +++ b/version.py @@ -1 +1 @@ -VERSION = "0.10.11" +VERSION = "0.10.12"