fix(linkedin): scroll the real container so profile-read sees Experience and Education (#417)

plugins/linkedin/profile-read.js

@@ -5,9 +5,13 @@ import {
   assertSafeLinkedinUrl,
   compactRepeatedText,
   normalizeWhitespace,
+  scrollToSections,
   unwrapEvaluateResult,
 } from './shared.js';
 
+/** Sections that only exist in the DOM once the profile has been scrolled. */
+const LAZY_PROFILE_SECTIONS = ['about', 'experience', 'education', 'featured'];
+
 function normalizeProfileReadUrl(value) {
   const url = assertSafeLinkedinUrl(value || 'https://www.linkedin.com/in/me/', 'profile-url', '/in/me/');
   const parsed = new URL(url);
@@ -125,7 +129,10 @@ cli({
     await page.goto(profileUrl);
     await page.wait(5);
     await assertLinkedInAuthenticated(page, 'LinkedIn profile-read');
+    // Kept for older layouts that scroll the window; the current profile UI
+    // scrolls inside main#workspace, which autoScroll cannot move.
     await page.autoScroll({ times: 4, delayMs: 700 });
+    await scrollToSections(page, LAZY_PROFILE_SECTIONS);
     await page.wait(1);
     const row = unwrapEvaluateResult(await page.evaluate(buildProfileExtractionScript()));
     let aboutEdit = {};

plugins/linkedin/shared.js

@@ -122,3 +122,96 @@ export async function assertLinkedInAuthenticated(page, context) {
 export function splitVisibleLines(text) {
   return String(text || '').split(/\n+/).map(normalizeWhitespace).filter(Boolean);
 }
+
+/**
+ * Step the element that actually scrolls the current LinkedIn layout.
+ *
+ * `page.autoScroll()` drives `window.scrollTo`, but the current profile UI
+ * scrolls inside `main#workspace`, so the window scroller never moves and the
+ * lazy loaders for later sections (Experience, Education, ...) never fire. This
+ * script advances the real scroll container — falling back to the window
+ * scroller on older layouts — and reports which of the requested section
+ * headings are present so the caller can stop as soon as they have loaded.
+ */
+export function buildSectionScrollScript(headings) {
+  const wanted = JSON.stringify((headings || []).map((value) => String(value).toLowerCase()));
+  return String.raw`(() => {
+    const clean = (s) => String(s || '').replace(/[\u00a0\u202f]+/g, ' ').replace(/\s+/g, ' ').trim();
+    const wanted = ${wanted};
+    const seen = Array.from(document.querySelectorAll('main h2, main h3, section h2, section h3'))
+      .map((el) => clean(el.innerText || el.textContent || '').toLowerCase())
+      .filter(Boolean);
+    const found = wanted.filter((name) => seen.includes(name));
+    const scrollable = (el) => {
+      if (!el) return false;
+      if (el.scrollHeight <= el.clientHeight + 1) return false;
+      const overflowY = (window.getComputedStyle ? window.getComputedStyle(el).overflowY : '') || '';
+      return overflowY === 'auto' || overflowY === 'scroll' || overflowY === 'overlay';
+    };
+    const findScroller = () => {
+      const workspace = document.querySelector('main#workspace') || document.querySelector('#workspace');
+      if (workspace && workspace.scrollHeight > workspace.clientHeight + 1) return workspace;
+      let node = document.querySelector('main');
+      while (node && node !== document.body && node !== document.documentElement) {
+        if (scrollable(node)) return node;
+        node = node.parentElement;
+      }
+      return null;
+    };
+    const scroller = findScroller();
+    if (scroller) {
+      const bottom = Math.max(0, scroller.scrollHeight - scroller.clientHeight);
+      const step = Math.max(scroller.clientHeight || 0, 400);
+      scroller.scrollTop = Math.min(bottom, scroller.scrollTop + step);
+      return {
+        found,
+        atEnd: scroller.scrollTop >= bottom - 2,
+        container: scroller.id ? '#' + scroller.id : String(scroller.tagName || '').toLowerCase(),
+      };
+    }
+    const doc = document.documentElement;
+    const bottom = Math.max(0, doc.scrollHeight - window.innerHeight);
+    const step = Math.max(window.innerHeight || 0, 400);
+    window.scrollTo(0, Math.min(bottom, (window.scrollY || 0) + step));
+    return {
+      found,
+      atEnd: (window.scrollY || 0) >= bottom - 2,
+      container: 'window',
+    };
+  })()`;
+}
+
+/**
+ * Scroll until every requested section heading is loaded, the container stops
+ * growing, or `rounds` is exhausted. One `atEnd` round is not the end: LinkedIn
+ * lazy-loads on reaching the bottom, so the container grows and the next round
+ * has further to travel. Two consecutive `atEnd` rounds with no newly found
+ * section mean the content really is exhausted.
+ *
+ * Never throws — a layout this helper cannot scroll is not itself a command
+ * failure, only a reason the extraction below it may see fewer sections.
+ */
+export async function scrollToSections(page, headings, options = {}) {
+  const rounds = options.rounds ?? 8;
+  const waitSeconds = options.waitSeconds ?? 1;
+  const targets = (headings || []).map((value) => String(value).toLowerCase());
+  let last = { found: [], atEnd: false, container: '' };
+  let foundCount = 0;
+  let endRounds = 0;
+  for (let round = 0; round < rounds; round++) {
+    let payload;
+    try {
+      payload = unwrapEvaluateResult(await page.evaluate(buildSectionScrollScript(targets)));
+    } catch {
+      return last;
+    }
+    if (payload && typeof payload === 'object') last = payload;
+    const found = Array.isArray(last.found) ? last.found : [];
+    if (targets.every((name) => found.includes(name))) return last;
+    endRounds = last.atEnd === true && found.length === foundCount ? endRounds + 1 : 0;
+    foundCount = found.length;
+    if (endRounds >= 2) return last;
+    await page.wait(waitSeconds);
+  }
+  return last;
+}

plugins/linkedin/test/profile-read.test.js

@@ -53,25 +53,59 @@ describe('linkedin profile-read adapter', () => {
     });
   });
 
-  it('does not require edit access when reading an explicit profile URL', async () => {
-    const page = {
+  const workspacePage = (row, scrollRounds) => {
+    const rounds = [...scrollRounds];
+    const evaluated = [];
+    return {
+      evaluated,
       goto: vi.fn(async () => {}),
       wait: vi.fn(async () => {}),
       autoScroll: vi.fn(async () => {}),
-      evaluate: vi.fn()
-        .mockResolvedValueOnce(false)
-        .mockResolvedValueOnce({
-          profile_url: 'https://www.linkedin.com/in/alice/',
-          name: 'Alice',
-          headline: 'Engineer',
-          about: 'Builds products',
-        }),
+      evaluate: vi.fn(async (script) => {
+        evaluated.push(script);
+        if (script.includes('authwall')) return false;
+        if (script.includes('scrollTop')) return rounds.shift() ?? { found: [], atEnd: true };
+        return row;
+      }),
     };
+  };
+
+  it('does not require edit access when reading an explicit profile URL', async () => {
+    const page = workspacePage({
+      profile_url: 'https://www.linkedin.com/in/alice/',
+      name: 'Alice',
+      headline: 'Engineer',
+      about: 'Builds products',
+    }, [{ found: ['about', 'experience', 'education', 'featured'], atEnd: false }]);
 
     await expect(command.func(page, { 'profile-url': 'https://www.linkedin.com/in/alice/' }))
       .resolves.toMatchObject([{ name: 'Alice', about: 'Builds products' }]);
     expect(page.goto).toHaveBeenCalledTimes(1);
     expect(page.goto).toHaveBeenCalledWith('https://www.linkedin.com/in/alice/');
     expect(page.goto.mock.calls.some(([url]) => String(url).includes('/edit/forms/'))).toBe(false);
   });
+
+  it('scrolls the inner container until the lazy sections load before extracting', async () => {
+    const page = workspacePage({
+      profile_url: 'https://www.linkedin.com/in/alice/',
+      name: 'Alice',
+      experience: 'Engineer at Acme',
+      education: 'Example University',
+    }, [
+      { found: [], atEnd: false, container: '#workspace' },
+      { found: ['about'], atEnd: false, container: '#workspace' },
+      { found: ['about', 'experience', 'education', 'featured'], atEnd: false, container: '#workspace' },
+    ]);
+
+    await expect(command.func(page, { 'profile-url': 'https://www.linkedin.com/in/alice/' }))
+      .resolves.toMatchObject([{ experience: 'Engineer at Acme', education: 'Example University' }]);
+
+    // window autoScroll stays for older layouts, but cannot move main#workspace
+    expect(page.autoScroll).toHaveBeenCalledWith({ times: 4, delayMs: 700 });
+    const scrollScripts = page.evaluated.filter((script) => script.includes('scrollTop'));
+    expect(scrollScripts).toHaveLength(3);
+    expect(scrollScripts[0]).toContain("main#workspace");
+    // the extraction runs only after the scroll rounds
+    expect(page.evaluated.at(-1)).toContain('readSection');
+  });
 });

plugins/linkedin/test/shared-scroll.test.js

@@ -0,0 +1,137 @@
+import { describe, expect, it, vi } from 'vitest';
+import { JSDOM } from 'jsdom';
+import { buildSectionScrollScript, scrollToSections } from '../shared.js';
+
+/**
+ * Runs the generated script against a JSDOM document. JSDOM reports every
+ * layout metric as 0, so the fixture defines scrollHeight/clientHeight
+ * explicitly and a settable scrollTop, which is exactly the state the script
+ * reads to decide which element scrolls.
+ */
+function runScript(html, { headings, scrollHeight = 4000, clientHeight = 800, overflowY = 'auto' } = {}) {
+  const dom = new JSDOM(html, { url: 'https://www.linkedin.com/in/alice/', runScripts: 'outside-only' });
+  const { window } = dom;
+  const workspace = window.document.querySelector('#workspace');
+  if (workspace) {
+    Object.defineProperty(workspace, 'scrollHeight', { value: scrollHeight, configurable: true });
+    Object.defineProperty(workspace, 'clientHeight', { value: clientHeight, configurable: true });
+    let top = 0;
+    Object.defineProperty(workspace, 'scrollTop', {
+      get: () => top,
+      set: (value) => { top = value; },
+      configurable: true,
+    });
+    workspace.style.overflowY = overflowY;
+  }
+  Object.defineProperty(window.document.documentElement, 'scrollHeight', { value: scrollHeight, configurable: true });
+  window.innerHeight = clientHeight;
+  const scrolled = [];
+  window.scrollTo = (x, y) => { scrolled.push(y); window.scrollY = y; };
+  const result = window.eval(buildSectionScrollScript(headings ?? ['experience', 'education']));
+  return { result, window, workspace, scrolled };
+}
+
+describe('linkedin section scrolling', () => {
+  it('scrolls main#workspace when the window scroller cannot move', () => {
+    const { result, workspace, scrolled } = runScript(`
+      <main id="workspace" style="overflow-y: auto">
+        <section><h2>About</h2></section>
+      </main>
+    `);
+
+    expect(result.container).toBe('#workspace');
+    expect(workspace.scrollTop).toBe(800);
+    expect(result.atEnd).toBe(false);
+    expect(result.found).toEqual([]);
+    expect(scrolled).toEqual([]);
+  });
+
+  it('reports the requested headings once they are in the DOM', () => {
+    const { result } = runScript(`
+      <main id="workspace" style="overflow-y: auto">
+        <section><h2>Experience</h2></section>
+        <section><h2>Education</h2></section>
+      </main>
+    `);
+
+    expect(result.found).toEqual(['experience', 'education']);
+  });
+
+  it('falls back to the window scroller on layouts without an inner container', () => {
+    const { result, scrolled } = runScript(`
+      <main>
+        <section><h2>Experience</h2></section>
+      </main>
+    `);
+
+    expect(result.container).toBe('window');
+    expect(scrolled).toEqual([800]);
+    expect(result.found).toEqual(['experience']);
+  });
+
+  it('reports the end of the container so callers stop scrolling', () => {
+    const { result } = runScript(`
+      <main id="workspace" style="overflow-y: auto"><section><h2>About</h2></section></main>
+    `, { scrollHeight: 1000, clientHeight: 900 });
+
+    expect(result.container).toBe('#workspace');
+    expect(result.atEnd).toBe(true);
+  });
+});
+
+describe('scrollToSections', () => {
+  const page = (payloads) => ({
+    evaluate: vi.fn().mockImplementation(async () => payloads.shift() ?? { found: [], atEnd: true }),
+    wait: vi.fn().mockResolvedValue(undefined),
+  });
+
+  it('keeps scrolling until every requested section has loaded', async () => {
+    const target = page([
+      { found: [], atEnd: false, container: '#workspace' },
+      { found: ['experience'], atEnd: false, container: '#workspace' },
+      { found: ['experience', 'education'], atEnd: false, container: '#workspace' },
+    ]);
+
+    await expect(scrollToSections(target, ['experience', 'education']))
+      .resolves.toMatchObject({ found: ['experience', 'education'] });
+    expect(target.evaluate).toHaveBeenCalledTimes(3);
+  });
+
+  it('stops once the container is stably at its end and no section appeared', async () => {
+    const target = page([
+      { found: [], atEnd: false, container: '#workspace' },
+      { found: [], atEnd: true, container: '#workspace' },
+      { found: [], atEnd: true, container: '#workspace' },
+    ]);
+
+    await expect(scrollToSections(target, ['experience'])).resolves.toMatchObject({ atEnd: true });
+    expect(target.evaluate).toHaveBeenCalledTimes(3);
+  });
+
+  it('keeps scrolling past an atEnd round that lazy-loaded a new section', async () => {
+    const target = page([
+      { found: [], atEnd: true, container: '#workspace' },
+      { found: ['experience'], atEnd: true, container: '#workspace' },
+      { found: ['experience', 'education'], atEnd: true, container: '#workspace' },
+    ]);
+
+    await expect(scrollToSections(target, ['experience', 'education']))
+      .resolves.toMatchObject({ found: ['experience', 'education'] });
+    expect(target.evaluate).toHaveBeenCalledTimes(3);
+  });
+
+  it('gives up after the round budget instead of scrolling forever', async () => {
+    const target = page([]);
+    target.evaluate.mockResolvedValue({ found: [], atEnd: false, container: '#workspace' });
+
+    await scrollToSections(target, ['experience'], { rounds: 3 });
+
+    expect(target.evaluate).toHaveBeenCalledTimes(3);
+  });
+
+  it('never fails the command when the page cannot be evaluated', async () => {
+    const target = { evaluate: vi.fn().mockRejectedValue(new Error('page closed')), wait: vi.fn() };
+
+    await expect(scrollToSections(target, ['experience'])).resolves.toMatchObject({ found: [] });
+  });
+});