feat(map-tokens): kind-scope the candidate list in the prompt (#201)

Author: tylerpina

packages/experience-design-system-generation/skills/map-tokens.md

@@ -13,7 +13,7 @@ This is a narrowing fallback, not a classification step. It does not re-classify
 All input is embedded inline in the prompt before this file:
 
 - **Generated CDF so far** — design-category, token-typed props only, grouped by component. Every prop shown here already has `$type: "token"` and `$category: "design"`; you do not need to re-verify either. A prop that already carries a `$token.allowed` entry was resolved earlier from source evidence — leave it alone. Only decide for props that arrive with no `$token.allowed` at all.
-- **Token path index** — a flat array of `{ "path": "<dot.notation.path>", "type": "<DTCG $type>" }` covering every leaf token in the library. **No `$value` is included** — the mapping decision only needs paths and their DTCG type, not their concrete values. A prop's candidates are always the subset of this index whose `type` matches the prop's `$token.kind` — ignore every entry outside that type.
+- **Token path index** — one or more sections, each a flat list of `path · type` lines (one leaf token per line, `$value` omitted). Every section is already pre-scoped: a section titled with a `$token.kind` (e.g. "color candidates only") contains every leaf token of that type and no others; a prop with that `$token.kind` draws its candidates only from the matching section. The "full tree" section — present only when at least one prop has no `$token.kind` — lists every leaf token in the library, unscoped, exactly as before. Never mix candidates across sections; a prop's candidates come from exactly one.
 - **Component source references** — the real file text for each component (bounded/truncated), rendered inline as a fenced code block, so you can look for an explicit restriction: a comment naming the valid tokens, or code that validates the prop against a fixed list of token paths. A default value or a `tokenReference` is the prop's *default*, not a restriction — see the decision tree. **You have no filesystem access and no tools — `sourcePath` is a citation label only, never something to open.** When a component's source couldn't be read (moved/deleted since extraction), it's listed separately by path with no code block; for those, there is no source evidence to narrow from — skip the prop.
 
 ```typescript
@@ -75,7 +75,7 @@ Emit one JSON object per line. The CLI parses lines starting with `{`. Lines not
 - Emit exactly one JSON object per line. No multi-line JSON.
 - Only emit a call for a prop that appears in the "Generated CDF so far" section, and only when it arrived **without** an existing `$token.allowed`.
 - `token_allowed` is required and must be non-empty. Never emit a call with an empty or missing `token_allowed` — if there's nothing to narrow to, emit no call at all.
-- Every path in `token_allowed` must be an individual **leaf** path that exists verbatim in the token path index, and must be of the prop's `$token.kind` type. Never emit a group/prefix path (e.g. `colors.brand`) — the index has no entry for groups, only leaves. Never invent a path, and never include a variant/enum name in place of a real token path — if you can't find a matching path, omit that entry.
+- Every path in `token_allowed` must be an individual **leaf** path that exists verbatim in the token path index section matching the prop's `$token.kind` (or the "full tree" section, for a prop with no `$token.kind`). Never emit a group/prefix path (e.g. `colors.brand`) — the index has no entry for groups, only leaves. Never invent a path, and never include a variant/enum name in place of a real token path — if you can't find a matching path, omit that entry.
 - `description` is a short internal rationale for the developer reviewing the import — not customer-facing copy.
 - No `$value` is available in this step. Reason from paths and `$type` only.
 

packages/experience-design-system-generation/src/prompt-builder.ts

@@ -210,6 +210,44 @@ function buildTokenPathIndex(tree: TokenTree, prefix = ''): TokenPathIndexEntry[
   return entries;
 }
 
+/** One line per candidate, e.g. `colors.brand.primary · color` — legible and countable, unlike nested JSON. */
+function formatTokenCandidateLines(entries: TokenPathIndexEntry[]): string {
+  return entries.map((entry) => `${entry.path} · ${entry.type}`).join('\n');
+}
+
+/**
+ * Distinct `$token.kind` values among design-category, token-typed props in
+ * `cdf`, plus whether any such prop has no `$token.kind` at all. Mirrors
+ * `filterDesignTokenProps`'s notion of a relevant prop without requiring a
+ * second pass over the raw CDF.
+ */
+function collectTokenKinds(cdf: GeneratedCdf): { kinds: string[]; hasUnscoped: boolean } {
+  const kinds = new Set<string>();
+  let hasUnscoped = false;
+  for (const component of Object.values(cdf)) {
+    const properties = component.$properties;
+    if (!properties) continue;
+    for (const prop of Object.values(properties)) {
+      if (prop.$type !== 'token' || prop.$category !== 'design') continue;
+      const kind = prop['$token.kind'];
+      if (typeof kind === 'string' && kind.length > 0) {
+        kinds.add(kind);
+      } else {
+        hasUnscoped = true;
+      }
+    }
+  }
+  return { kinds: [...kinds].sort(), hasUnscoped };
+}
+
+/** Renders one kind-scoped (or, for `kind: null`, full-tree) candidate section. Sections are cumulative, never merged, so each stays independently legible. */
+function renderTokenCandidateSection(kind: string | null, entries: TokenPathIndexEntry[]): string {
+  const label = kind
+    ? `Token path index — ${kind} candidates only`
+    : 'Token path index — full tree (no $token.kind to scope by)';
+  return `${label}, one leaf token per line as \`path · type\`, no \`$value\`:\n${formatTokenCandidateLines(entries)}`;
+}
+
 function buildPreamble(options: PromptOptions): string {
   const {
     skill,
@@ -248,11 +286,18 @@ function buildPreamble(options: PromptOptions): string {
     }
   }
   if (tokenTree) {
-    const index = buildTokenPathIndex(tokenTree);
+    const index = buildTokenPathIndex(tokenTree).sort((a, b) => a.path.localeCompare(b.path));
     if (index.length > 0) {
-      sections.push(
-        `Token path index — path and $type only, no $value (JSON):\n\`\`\`json\n${JSON.stringify(index)}\n\`\`\``,
-      );
+      if (generatedCdf) {
+        const { kinds, hasUnscoped } = collectTokenKinds(generatedCdf);
+        for (const kind of kinds) {
+          const scoped = index.filter((entry) => entry.type === kind);
+          if (scoped.length > 0) sections.push(renderTokenCandidateSection(kind, scoped));
+        }
+        if (hasUnscoped) sections.push(renderTokenCandidateSection(null, index));
+      } else {
+        sections.push(renderTokenCandidateSection(null, index));
+      }
     }
   }
   if (componentSourceRefs && componentSourceRefs.length > 0) {
@@ -399,9 +444,9 @@ The one tool call you may emit:
 Rules:
 - Emit exactly one JSON object per line. No multi-line JSON. No markdown fences around the lines.
 - Only emit a call for a prop that appears in the "Generated CDF so far" section and does not already carry \`$token.allowed\` — a prop that already has one was resolved from source and must not be contradicted.
-- Candidates are scoped to the prop's \`$token.kind\` — only tokens of that type from the "Token path index" can bind; ignore every other entry.
-- \`token_allowed\` is a flat list of individual **leaf** token paths — never a group/prefix path. The "Token path index" contains one entry per leaf token only; a path like \`colors.brand\` that groups \`colors.brand.primary\`/\`colors.brand.secondary\` does NOT itself appear in the index and must never be emitted.
-- Every path in \`token_allowed\` must exist verbatim in the "Token path index" section and match the prop's \`$token.kind\`. Never invent a path, and never substitute a variant/enum name for a real token path. If a path you'd otherwise suggest is missing from the index, omit it rather than guessing.
+- Each "Token path index" section below is already scoped to one \`$token.kind\` — a prop only draws candidates from the section matching its own \`$token.kind\`. A prop with no \`$token.kind\` draws from the "full tree" section instead. Never cross sections.
+- \`token_allowed\` is a flat list of individual **leaf** token paths — never a group/prefix path. Each "Token path index" section contains one entry per leaf token only; a path like \`colors.brand\` that groups \`colors.brand.primary\`/\`colors.brand.secondary\` does NOT itself appear in any section and must never be emitted.
+- Every path in \`token_allowed\` must exist verbatim in the matching "Token path index" section and match the prop's \`$token.kind\`. Never invent a path, and never substitute a variant/enum name for a real token path. If a path you'd otherwise suggest is missing from the index, omit it rather than guessing.
 - \`token_allowed\` is required and must be non-empty when the call is emitted. Restriction requires explicit evidence: a comment naming the valid tokens, or code that validates the prop against a fixed list of token paths.
 - A default value or a \`tokenReference\` is the prop's default, not a restriction. On its own it yields no call — narrowing to that one path would leave the marketer a single option. If you narrow on other evidence, the default's path must be in the list.
 - A token prop whose type is a union of variant names (\`'primary' | 'secondary'\`) is misclassified — it should be an enum. Do not narrow it; emit nothing and flag it in a prose line.

packages/experience-design-system-generation/test/prompt-builder.test.ts

@@ -281,21 +281,123 @@ describe('buildPrompt', () => {
       expect(prompt).not.toContain('Widget');
     });
 
-    it('flattens the token tree to a path + $type index with no $value', async () => {
+    it('flattens the token tree to one path · type line per candidate, no $value', async () => {
       const prompt = await buildPrompt({
         skill: 'map-tokens',
         mode: 'autonomous',
         generatedCdf: GENERATED_CDF,
         tokenTree: TOKEN_TREE,
         outDir: '/fake/out',
       });
-      expect(prompt).toContain('{"path":"colors.surface.default","type":"color"}');
-      expect(prompt).toContain('{"path":"colors.brand.primary","type":"color"}');
+      expect(prompt).toContain('colors.surface.default · color');
+      expect(prompt).toContain('colors.brand.primary · color');
       expect(prompt).toContain('Token path index');
       expect(prompt).not.toContain('#ffffff');
       expect(prompt).not.toContain('#0066ff');
     });
 
+    it('kind-scopes the candidate list per property, excluding tokens of other kinds', async () => {
+      const cdf = {
+        Card: {
+          $type: 'component',
+          $properties: {
+            bgColor: { $type: 'token', $category: 'design', '$token.kind': 'color' },
+          },
+        },
+      };
+      const tree = {
+        colors: { brand: { primary: { $type: 'color', $value: '#0066ff' } } },
+        spacing: { md: { $type: 'dimension', $value: '16px' } },
+      };
+      const prompt = await buildPrompt({
+        skill: 'map-tokens',
+        mode: 'autonomous',
+        generatedCdf: cdf,
+        tokenTree: tree,
+        outDir: '/fake/out',
+      });
+      expect(prompt).toContain('color candidates only');
+      expect(prompt).toContain('colors.brand.primary · color');
+      expect(prompt).not.toContain('spacing.md · dimension');
+      expect(prompt).not.toContain('Token path index — full tree');
+    });
+
+    it('falls back to the full, unscoped tree for a prop with no $token.kind', async () => {
+      const cdf = {
+        Card: {
+          $type: 'component',
+          $properties: {
+            bgColor: { $type: 'token', $category: 'design' },
+          },
+        },
+      };
+      const tree = {
+        colors: { brand: { primary: { $type: 'color', $value: '#0066ff' } } },
+        spacing: { md: { $type: 'dimension', $value: '16px' } },
+      };
+      const prompt = await buildPrompt({
+        skill: 'map-tokens',
+        mode: 'autonomous',
+        generatedCdf: cdf,
+        tokenTree: tree,
+        outDir: '/fake/out',
+      });
+      expect(prompt).toContain('full tree');
+      expect(prompt).toContain('colors.brand.primary · color');
+      expect(prompt).toContain('spacing.md · dimension');
+    });
+
+    it('renders a separate scoped section per distinct $token.kind, plus the full-tree fallback when mixed', async () => {
+      const cdf = {
+        Card: {
+          $type: 'component',
+          $properties: {
+            bgColor: { $type: 'token', $category: 'design', '$token.kind': 'color' },
+            gap: { $type: 'token', $category: 'design', '$token.kind': 'dimension' },
+            unscopedProp: { $type: 'token', $category: 'design' },
+          },
+        },
+      };
+      const tree = {
+        colors: { brand: { primary: { $type: 'color', $value: '#0066ff' } } },
+        spacing: { md: { $type: 'dimension', $value: '16px' } },
+      };
+      const prompt = await buildPrompt({
+        skill: 'map-tokens',
+        mode: 'autonomous',
+        generatedCdf: cdf,
+        tokenTree: tree,
+        outDir: '/fake/out',
+      });
+      expect(prompt).toContain('color candidates only');
+      expect(prompt).toContain('dimension candidates only');
+      expect(prompt).toContain('full tree');
+      const colorIdx = prompt.indexOf('color candidates only');
+      const dimensionIdx = prompt.indexOf('dimension candidates only');
+      expect(colorIdx).toBeGreaterThanOrEqual(0);
+      expect(dimensionIdx).toBeGreaterThan(colorIdx);
+    });
+
+    it('produces an identical prompt across repeated calls with the same input (deterministic ordering)', async () => {
+      const cdf = {
+        Widget: {
+          $type: 'component',
+          $properties: {
+            gap: { $type: 'token', $category: 'design', '$token.kind': 'dimension' },
+            color: { $type: 'token', $category: 'design', '$token.kind': 'color' },
+          },
+        },
+      };
+      const tree = {
+        spacing: { md: { $type: 'dimension', $value: '16px' }, sm: { $type: 'dimension', $value: '8px' } },
+        colors: { brand: { primary: { $type: 'color', $value: '#0066ff' } } },
+      };
+      const buildOnce = () =>
+        buildPrompt({ skill: 'map-tokens', mode: 'autonomous', generatedCdf: cdf, tokenTree: tree, outDir: '/fake/out' });
+      const [first, second] = await Promise.all([buildOnce(), buildOnce()]);
+      expect(first).toEqual(second);
+    });
+
     it('falls back to a path-only listing when content could not be read', async () => {
       const prompt = await buildPrompt({
         skill: 'map-tokens',
@@ -392,7 +494,7 @@ describe('buildPrompt', () => {
         outDir: '/fake/out',
       });
       expect(prompt).not.toContain('design-category token props only (JSON)');
-      expect(prompt).not.toContain('Token path index — path and $type only');
+      expect(prompt).not.toContain('Token path index —');
       expect(prompt).not.toContain('### Component source references');
       expect(prompt).not.toContain('Component source unavailable for');
     });