Skip to content

Characters: a caption for each picture, up to 24 pictures and an editable description #564

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
albertleal wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Characters: a caption for each picture, up to 24 pictures and an editable description #564

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 18 additions & 8 deletions freevideo_engine/agent/cast.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
ORIGINS = ('user', 'agent', 'builtin')
LABELS = schema.LABELS
EDITABLE = ('name', 'description', 'images', 'voice')
MAX_IMAGES = 6
MAX_IMAGES = schema.MAX_CAST_IMAGES
MAX_CANDIDATES = 6
MAX_RECEIPTS = 64
VOICE_SECONDS = (2, 15)
Expand All @@ -40,7 +40,7 @@
CLOSE_UP = dict(aspect='1:1', seconds=4)
SCENE = dict(aspect='16:9', seconds=6)

# The sheet of every view a shot carries for each person, put together here from the card's own pictures: up to six
# The sheet of every view a continuing shot carries for a single person, put together here from the card's own pictures: up to six
# equal cells (three to a row) on a light grey ground, each picture scaled whole and centred, no text.
SHEET_VERSION = 1
SHEET_ORDER = ('front', 'three_quarter', 'side', 'back', 'face') # then any other label, in card order
Expand Down Expand Up @@ -140,6 +140,12 @@ def _plain(item):
return None


def _caption(row):
"""A requested picture's caption on one line, as picture fields ({} when it has none)."""
text = ' '.join(str(row.get('caption') or '').split())
return {'caption': text} if text else {}


def _check(record, stem):
card = record['card']
if (card['id'] != stem or not CARD.fullmatch(stem) or card['kind'] not in KINDS
Expand All @@ -161,7 +167,8 @@ def _check(record, stem):
if not isinstance(card[key], list) or len(card[key]) > limit:
raise ValueError('Invalid pictures')
for row in card[key]:
if not isinstance(row['label'], str) or not isinstance(row['media']['file'], str):
if (not isinstance(row['label'], str) or not isinstance(row['media']['file'], str)
or not isinstance(row.get('caption', ''), str)):
raise ValueError('Invalid picture')
unknown_labels |= row['label'] not in LABELS
if card['voice'] is not None:
Expand Down Expand Up @@ -383,7 +390,7 @@ def create(self, value, *, by='builtin', ref=None, made_for=None, local=True, cl
images = []
for index, row in enumerate(value.get('images', [])):
entry = self._picture_at(identity, row['media'], by, index)
images.append(dict(entry, label=row.get('label', 'other')))
images.append(dict(entry, label=row.get('label', 'other'), **_caption(row)))
voice = self._voice(identity, value['voice']) if value.get('voice') else None
with self.changed:
digest, replay = self._receipt('cast:create', value, ref, client)
Expand Down Expand Up @@ -461,7 +468,7 @@ def update(self, identity, value, *, by='builtin', ref=None, revision=None, loca
for key, rows in changes.items():
if key == 'images':
resolved['images'] = [entry if entry is not None else dict(self._picture_at(identity, row['media'], by, index),
label=row.get('label', 'other')) for index, (row, entry) in enumerate(rows)]
label=row.get('label', 'other'), **_caption(row)) for index, (row, entry) in enumerate(rows)]
elif key == 'voice':
resolved['voice'] = None if rows is None else self._voice(identity, rows)
else:
Expand Down Expand Up @@ -510,16 +517,19 @@ def _changes(self, card, value, by):
entry.pop('label_guess', None)
if by == 'user':
entry.pop('generated', None) # The user's label makes it the user's picture.
if 'caption' in row: # Left out, the caption stays; "" removes it.
entry.pop('caption', None)
entry.update(_caption(row))
else:
fresh = True
rows.append((row, entry))
current = [(r['media']['file'], r['label']) for r in card['images']]
proposed = [(e['media']['file'], e['label']) for _, e in rows if e is not None]
current = [(r['media']['file'], r['label'], r.get('caption')) for r in card['images']]
proposed = [(e['media']['file'], e['label'], e.get('caption')) for _, e in rows if e is not None]
if fresh or proposed != current:
changes['images'] = rows
if by != 'user' and card['edited_by_user']['images']:
# The user's pictures may only be joined by candidates the server made; never replaced.
untouched = all((f, l) in proposed for f, l in current) and len(kept) == len(current)
untouched = all(row in proposed for row in current) and len(kept) == len(current)
if fresh or not untouched or not added:
raise self._by_user(card, 'images')
if 'voice' in value:
Expand Down
50 changes: 37 additions & 13 deletions freevideo_engine/agent/drama.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,8 @@ def __init__(self, problems):
CAPTIONED = 3 # the score of a take with unwanted captions
CARD = re.compile(r'c[0-9a-f]{12}') # characters.<key>.card names a cast library card
# How a shot carries its cast cards. Change these after the reference tests; nothing else needs to.
# Each person is two pictures: one sheet of every view (made by the cast library when the card has none), then
# the face close-up. A continuing shot with several people has no room for two each: it carries one picture a person.
# A fresh shot carries every picture of each person, the one standing for the person last. A continuing shot has room
# for three pictures: one person carries the sheet of every view then the face, several people one picture each.
FACE_ORDER = ('face', 'front', 'three_quarter') # the one picture standing for a person, then any other label, in card order
MAX_VOICES = 3 # voice clips per shot, the model limit
MAX_REFERENCES = 32
Expand Down Expand Up @@ -266,7 +266,8 @@ def prompt(play, shot):
def subject_definitions(subjects):
"""The prompt prefix tying cast cards to their reference tags; the one place its wording lives.

subjects: [{'name', 'description', 'pictures': (first, last), a list of such pairs, or None, 'audio': number or None}].
subjects: [{'name', 'description', 'pictures': (first, last), a list of such pairs, or None, 'captions': [(picture,
text)], 'audio': number or None}].
The page writes the same prompt from the same wording (web/cast_mention.js); tests/fixtures/subject_definitions.json
holds the cases both sides must reproduce exactly."""
parts = []
Expand All @@ -282,6 +283,10 @@ def subject_definitions(subjects):
for a, b in ranges)
single = len(ranges) == 1 and ranges[0][0] == ranges[0][1]
tags.append('%s show%s its appearance' % (pictures, 's' if single else ''))
for picture, caption in subject.get('captions') or ():
caption = (caption or '').strip().rstrip('.')
if caption:
tags.append('<Picture %d> shows %s' % (picture, caption))
if subject.get('audio'):
tags.append('<Audio %d> is its voice' % subject['audio'])
if tags:
Expand All @@ -305,19 +310,30 @@ def sheet_of(card):


def person_pictures(card, *, alone=True, continues=False):
"""The pictures one person carries into a shot, in order: the sheet, then the face close-up (the front view
when there is no face). Two of them on every device. A card that has no sheet (one picture, or none could be
made) carries its face alone (else the front view). A shot that continues the previous one with several
people carries only the face (else the front view) of each, with no sheet."""
"""The pictures one person carries into a shot, in order. A fresh shot carries every picture of the card in card
order, the one standing for the person (the face, else the front view) moved last. A shot that continues the
previous one carries the sheet then that picture, or that picture alone when the card has no sheet (one picture,
or none could be made); with several people it carries only that picture of each (else the sheet)."""
views = _views(card)
if continues and not alone:
if not continues:
pictures = [row['media'] for row in card.get('images') or []]
if not views:
return pictures
return [media for media in pictures if media is not views[0]] + views[:1]
if not alone:
return views[:1] or ([sheet_of(card)] if sheet_of(card) else [])
sheet = sheet_of(card)
if sheet is None:
return views[:1]
return [sheet] + [view for view in views[:1] if view.get('file') != sheet.get('file')]


def picture_captions(card, pictures):
"""The card's caption of each of these pictures; None for one without a caption or not on the card (a sheet)."""
captions = {row['media'].get('file'): row.get('caption') for row in card.get('images') or []}
return [captions.get(item.get('file')) for item in pictures]


def cast_references(play, shot, cards, *, continues=False):
"""Which cast cards a shot carries, as references and a prompt prefix.

Expand All @@ -328,7 +344,8 @@ def cast_references(play, shot, cards, *, continues=False):
names its own first frame or references; an empty list means no pictures), 'missing' (a card was deleted),
'partial' (not everyone fit) or 'cards'. 'omitted' lists people left out
entirely; 'voice_omitted' lists people whose pictures are carried but whose
voice did not fit. A shot that continues the previous one carries no voice
voice did not fit. A person whose pictures do not all fit in the references
left carries the first ones and the one standing for the person. A shot that continues the previous one carries no voice
clip (its voices carry on from the previous shot) and at most CONTINUED_PICTURES
pictures: two for a single person, else one for each of up to that many people."""
characters = play.get('characters', {})
Expand Down Expand Up @@ -362,9 +379,12 @@ def cast_references(play, shot, cards, *, continues=False):
result['omitted'].append(key)
continue
voice = card.get('voice') is not None and voices < MAX_VOICES
if pictures + len(images) + voices + voice > MAX_REFERENCES:
room = MAX_REFERENCES - pictures - voices - voice
if room < min(len(images), 1):
result['omitted'].append(key)
continue
if len(images) > room:
images = images[:room - 1] + images[-1:]
if card.get('voice') is not None and not voice:
result['voice_omitted'].append(key) # shown, without the voice that did not fit
pictures, voices = pictures + len(images), voices + voice
Expand All @@ -381,6 +401,9 @@ def cast_references(play, shot, cards, *, continues=False):
subject = dict(name=person.get('name') or key, description=card.get('description'),
pictures=(picture + 1, picture + len(images)) if images else None,
audio=len(clips) if voice else None)
captions = [(picture + n, text) for n, text in enumerate(picture_captions(card, images), 1) if text]
if captions:
subject['captions'] = captions
picture += len(images)
if images or voice:
subjects.append(subject)
Expand All @@ -403,15 +426,16 @@ def cast_references(play, shot, cards, *, continues=False):
def with_cast(text, resolved, *, limit=PROMPT_LIMIT):
"""The submitted prompt: the subject definitions, then the shot's own prompt.

Cards can grow after a plan; when the whole prompt would pass the limit, the card descriptions are
shortened (the pictures still show each person). Plan previews and submissions both call this."""
Cards can grow after a plan; when the whole prompt would pass the limit, the card descriptions and picture
captions are shortened (the pictures still show each person). Plan previews and submissions both call this."""
if not resolved.get('prefix'):
return text
full = resolved['prefix'] + '\n\n' + text
if len(full) <= limit or not resolved.get('subjects'):
return full
for cut in DESCRIPTION_CUTS:
subjects = [dict(s, description=(s.get('description') or '')[:cut].rstrip() or None)
subjects = [dict(s, description=(s.get('description') or '')[:cut].rstrip() or None,
captions=[(n, text[:cut].rstrip()) for n, text in s.get('captions') or ()])
for s in resolved['subjects']]
short = subject_definitions(subjects) + '\n\n' + text
if len(short) <= limit:
Expand Down
12 changes: 8 additions & 4 deletions freevideo_engine/agent/guide.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
own prompt rules; the enhancement job applies the same contract.
"""
from ..default_prompt import DEFAULT_PROMPT
from .schema import CAST_OPERATIONS, DRAMA_OPERATIONS, IMAGE_OPERATIONS, PROJECT_OPERATIONS, SESSION_OPERATIONS, WHITEBOX_OPERATIONS
from .schema import CAST_OPERATIONS, DRAMA_OPERATIONS, IMAGE_OPERATIONS, MAX_CAST_IMAGES, PROJECT_OPERATIONS, SESSION_OPERATIONS, WHITEBOX_OPERATIONS

from .whitebox import SCENE_PROMPT

Expand Down Expand Up @@ -609,11 +609,15 @@ def connection_text(value):
GUIDE['cast_routes'] = [dict(operation=name, method=method, path=path, description=summary)
for name, (method, path, summary) in CAST_OPERATIONS.items()]
GUIDE['cast'] = [
'A card is a character or setting: name, a one-sentence English description used in prompts, up to six '
'labelled pictures (front, three_quarter, side, back, face, sheet, other) and an optional 2 to 15 s voice.',
'A card is a character or setting: name, a one-sentence English description used in prompts, up to %d '
'labelled pictures (front, three_quarter, side, back, face, sheet, other) and an optional 2 to 15 s voice. '
'Each picture may have a caption: what it shows that the others do not ("dragon tattoo on her back"), written '
'into the prompt beside its picture tag.'
% MAX_CAST_IMAGES,
'Changes need the revision you read (body revision or If-Match); stale_revision returns the current card in '
'error.card. Fields the user changed return changed_by_user: keep the user\'s version.',
'Film with a card by setting characters.<key>.card. Cards are read when each take is submitted. Shots that '
'Film with a card by setting characters.<key>.card. Cards are read when each take is submitted. A shot carries '
'every picture of its carded people within the 32 references (the face, else the front, last). Shots that '
'continue the previous one carry the sheet and the face for a single person, else one picture each (the face, '
'else the front) for at most 3 people, and keep the voices of the previous shot; POST /drama/plan shows each shot\'s cast. '
'A shot\'s own references replace the card pictures; references=[] sends none. '
Expand Down
4 changes: 2 additions & 2 deletions freevideo_engine/agent/productions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -978,8 +978,8 @@ def _cast(self, state, take, item):
return resolved

def _carry(self, play, shot, cards, item, *, continues):
"""The cast a shot carries: each person's sheet and face close-up on every device (one picture a person when
a continuing shot has several people). Plan previews and submissions both come here, so they agree."""
"""The cast a shot carries: every picture of each person (the sheet and face of one person, or one picture a
person, when the shot continues). Plan previews and submissions both come here, so they agree."""
return drama.cast_references(play, shot, cards, continues=continues)

def _current(self, play):
Expand Down
15 changes: 11 additions & 4 deletions freevideo_engine/agent/schema.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -592,6 +592,10 @@ def drama_schema(name):
return obj(dict(ref=REF, **({'transition': enum(('cut', 'crossfade'))} if name == 'finish' else {})))


# A card's pictures; each may carry a one-line caption that prompts write beside its <Picture N> tag.
MAX_CAST_IMAGES = 24
MAX_CAST_CAPTION = 200

CAST_OPERATIONS = {
'create': ('POST', '/cast', 'Create a character or setting card when no existing card fits'),
'list': ('GET', '/cast', 'Search the cast library before casting; reuse a matching card'),
Expand All @@ -612,7 +616,10 @@ def cast_schema(name, *, local=False):
sound['oneOf'][1]['properties']['url'] = picture['oneOf'][1]['properties']['url']
sound['oneOf'][1]['allOf'] = sound['oneOf'][1]['allOf'][:1]
fields = dict(name=string(120), description=nullable(string(1000)),
images=array(obj(dict(media=picture, label=enum(LABELS)), ('media',)), 0, 6),
images=array(obj(dict(media=picture, label=enum(LABELS), caption={'type': 'string', 'maxLength': MAX_CAST_CAPTION,
'description': 'Optional: what this picture shows, e.g. "dragon tattoo on her back"; written into prompts. '
'"" removes it.'}),
('media',)), 0, MAX_CAST_IMAGES),
voice=nullable(obj(dict(media=sound, start=number(0, 3600, default=0), end=number(0, 3600)), ('media',))))
if name == 'create':
return obj(dict(kind=enum(('character', 'setting'), default='character'), **fields, ref=REF), ('name',))
Expand All @@ -637,7 +644,7 @@ def cast_schema(name, *, local=False):
def cast_view():
picture = obj(dict(media=obj(dict(file=FILE, url=string(2048)), ('file', 'url')),
by=enum(('user', 'agent', 'builtin')), from_job=JOB, generated={'type': 'boolean'},
label={'type': 'string'}, label_guess={'type': 'boolean'}), ('media', 'by', 'label'))
label={'type': 'string'}, label_guess={'type': 'boolean'}, caption=string(MAX_CAST_CAPTION)), ('media', 'by', 'label'))
generating = obj(dict(job=JOB, face_job=nullable(JOB), jobs=array(JOB, 1, 5),
source=enum(('image', 'video')), replace={'type': 'boolean'}, at=string(),
by=enum(('user', 'agent', 'builtin'))), ('job', 'face_job', 'replace', 'at', 'by'))
Expand All @@ -646,7 +653,7 @@ def cast_view():
('job', 'face_job', 'seed', 'prompt_version'))
return dict(obj(dict(id=CAST, kind=enum(('character', 'setting')), name=string(120),
description=nullable(string(1000)), revision=number(1, integer=True),
images=array(picture, 0, 6), candidates=array(picture, 0, 6),
images=array(picture, 0, MAX_CAST_IMAGES), candidates=array(picture, 0, 6),
generating=nullable(generating), generated=nullable(generated),
generate_error=nullable(obj(dict(code=string(), job=JOB, message=string(), error_zh=string(), error_en=string()), ('code', 'job', 'message')))),
('id', 'kind', 'name', 'revision', 'images', 'candidates', 'generating')), additionalProperties=True)
Expand Down Expand Up @@ -1058,7 +1065,7 @@ def production_response():
cast_reference=CAST_REFERENCE, cast_omitted=dict(type='array', items=string()),
cast_voice_omitted=dict(type='array', items=string()),
cast_missing=dict(type='array', items=string()),
cast_pictures_each=dict(type=['integer', 'null'], description='Most pictures one person carries in this shot: two (the sheet of every view and the face close-up), or one each when a continuing shot has several people.'),
cast_pictures_each=dict(type=['integer', 'null'], description='Most pictures one person carries in this shot: every picture of the card (fewer only when the 32 references run out), or in a continuing shot the sheet and the face for one person, else one each.'),
takes=dict(type='array', items=dict(type='object', properties=dict(take=number(1, integer=True), job=REF,
task={'oneOf': [JOB, {'type': 'null'}]}, generation=number(0, integer=True), seed=number(0, integer=True),
stale={'type': 'boolean'}, error={'type': ['object', 'null']},
Expand Down
Loading