From 4a01df39200a9d9369b3a0f4cd44b844864890d4 Mon Sep 17 00:00:00 2001
From: Agnik47 <140933190+Agnik47@users.noreply.github.com>
Date: Fri, 28 Aug 2026 22:14:36 +0530
Subject: [PATCH] fix(linkedin): scroll the real container so profile-read sees
Experience and Education (#417)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
`page.autoScroll()` drives `window.scrollTo`, but LinkedIn's current profile UI
scrolls inside `main#workspace`. The window scroller never moves, the lazy
loaders for later sections never fire, and `profile-read` exits successfully
with empty `experience` and `education` while `profile-experience` returns the
same profile's entries.
`scrollToSections` now steps the element that actually scrolls — `main#workspace`,
else the nearest scrollable ancestor of `main`, else the window — and stops as
soon as the requested headings are in the DOM. One `atEnd` round is not the end:
LinkedIn lazy-loads on reaching the bottom, so the helper requires two
consecutive end rounds with no newly found section. `page.autoScroll()` stays
ahead of it for older window-scrolling layouts.
The helper never throws: a layout it cannot scroll is not a command failure.
---
plugins/linkedin/profile-read.js | 7 +
plugins/linkedin/shared.js | 93 +++++++++++++
plugins/linkedin/test/profile-read.test.js | 54 ++++++--
plugins/linkedin/test/shared-scroll.test.js | 137 ++++++++++++++++++++
4 files changed, 281 insertions(+), 10 deletions(-)
create mode 100644 plugins/linkedin/test/shared-scroll.test.js
diff --git a/plugins/linkedin/profile-read.js b/plugins/linkedin/profile-read.js
index 3a39e330..48d2a5c6 100644
--- a/plugins/linkedin/profile-read.js
+++ b/plugins/linkedin/profile-read.js
@@ -5,9 +5,13 @@ import {
assertSafeLinkedinUrl,
compactRepeatedText,
normalizeWhitespace,
+ scrollToSections,
unwrapEvaluateResult,
} from './shared.js';
+/** Sections that only exist in the DOM once the profile has been scrolled. */
+const LAZY_PROFILE_SECTIONS = ['about', 'experience', 'education', 'featured'];
+
function normalizeProfileReadUrl(value) {
const url = assertSafeLinkedinUrl(value || 'https://www.linkedin.com/in/me/', 'profile-url', '/in/me/');
const parsed = new URL(url);
@@ -125,7 +129,10 @@ cli({
await page.goto(profileUrl);
await page.wait(5);
await assertLinkedInAuthenticated(page, 'LinkedIn profile-read');
+ // Kept for older layouts that scroll the window; the current profile UI
+ // scrolls inside main#workspace, which autoScroll cannot move.
await page.autoScroll({ times: 4, delayMs: 700 });
+ await scrollToSections(page, LAZY_PROFILE_SECTIONS);
await page.wait(1);
const row = unwrapEvaluateResult(await page.evaluate(buildProfileExtractionScript()));
let aboutEdit = {};
diff --git a/plugins/linkedin/shared.js b/plugins/linkedin/shared.js
index bb8918f6..645ca8fa 100644
--- a/plugins/linkedin/shared.js
+++ b/plugins/linkedin/shared.js
@@ -122,3 +122,96 @@ export async function assertLinkedInAuthenticated(page, context) {
export function splitVisibleLines(text) {
return String(text || '').split(/\n+/).map(normalizeWhitespace).filter(Boolean);
}
+
+/**
+ * Step the element that actually scrolls the current LinkedIn layout.
+ *
+ * `page.autoScroll()` drives `window.scrollTo`, but the current profile UI
+ * scrolls inside `main#workspace`, so the window scroller never moves and the
+ * lazy loaders for later sections (Experience, Education, ...) never fire. This
+ * script advances the real scroll container — falling back to the window
+ * scroller on older layouts — and reports which of the requested section
+ * headings are present so the caller can stop as soon as they have loaded.
+ */
+export function buildSectionScrollScript(headings) {
+ const wanted = JSON.stringify((headings || []).map((value) => String(value).toLowerCase()));
+ return String.raw`(() => {
+ const clean = (s) => String(s || '').replace(/[\u00a0\u202f]+/g, ' ').replace(/\s+/g, ' ').trim();
+ const wanted = ${wanted};
+ const seen = Array.from(document.querySelectorAll('main h2, main h3, section h2, section h3'))
+ .map((el) => clean(el.innerText || el.textContent || '').toLowerCase())
+ .filter(Boolean);
+ const found = wanted.filter((name) => seen.includes(name));
+ const scrollable = (el) => {
+ if (!el) return false;
+ if (el.scrollHeight <= el.clientHeight + 1) return false;
+ const overflowY = (window.getComputedStyle ? window.getComputedStyle(el).overflowY : '') || '';
+ return overflowY === 'auto' || overflowY === 'scroll' || overflowY === 'overlay';
+ };
+ const findScroller = () => {
+ const workspace = document.querySelector('main#workspace') || document.querySelector('#workspace');
+ if (workspace && workspace.scrollHeight > workspace.clientHeight + 1) return workspace;
+ let node = document.querySelector('main');
+ while (node && node !== document.body && node !== document.documentElement) {
+ if (scrollable(node)) return node;
+ node = node.parentElement;
+ }
+ return null;
+ };
+ const scroller = findScroller();
+ if (scroller) {
+ const bottom = Math.max(0, scroller.scrollHeight - scroller.clientHeight);
+ const step = Math.max(scroller.clientHeight || 0, 400);
+ scroller.scrollTop = Math.min(bottom, scroller.scrollTop + step);
+ return {
+ found,
+ atEnd: scroller.scrollTop >= bottom - 2,
+ container: scroller.id ? '#' + scroller.id : String(scroller.tagName || '').toLowerCase(),
+ };
+ }
+ const doc = document.documentElement;
+ const bottom = Math.max(0, doc.scrollHeight - window.innerHeight);
+ const step = Math.max(window.innerHeight || 0, 400);
+ window.scrollTo(0, Math.min(bottom, (window.scrollY || 0) + step));
+ return {
+ found,
+ atEnd: (window.scrollY || 0) >= bottom - 2,
+ container: 'window',
+ };
+ })()`;
+}
+
+/**
+ * Scroll until every requested section heading is loaded, the container stops
+ * growing, or `rounds` is exhausted. One `atEnd` round is not the end: LinkedIn
+ * lazy-loads on reaching the bottom, so the container grows and the next round
+ * has further to travel. Two consecutive `atEnd` rounds with no newly found
+ * section mean the content really is exhausted.
+ *
+ * Never throws — a layout this helper cannot scroll is not itself a command
+ * failure, only a reason the extraction below it may see fewer sections.
+ */
+export async function scrollToSections(page, headings, options = {}) {
+ const rounds = options.rounds ?? 8;
+ const waitSeconds = options.waitSeconds ?? 1;
+ const targets = (headings || []).map((value) => String(value).toLowerCase());
+ let last = { found: [], atEnd: false, container: '' };
+ let foundCount = 0;
+ let endRounds = 0;
+ for (let round = 0; round < rounds; round++) {
+ let payload;
+ try {
+ payload = unwrapEvaluateResult(await page.evaluate(buildSectionScrollScript(targets)));
+ } catch {
+ return last;
+ }
+ if (payload && typeof payload === 'object') last = payload;
+ const found = Array.isArray(last.found) ? last.found : [];
+ if (targets.every((name) => found.includes(name))) return last;
+ endRounds = last.atEnd === true && found.length === foundCount ? endRounds + 1 : 0;
+ foundCount = found.length;
+ if (endRounds >= 2) return last;
+ await page.wait(waitSeconds);
+ }
+ return last;
+}
diff --git a/plugins/linkedin/test/profile-read.test.js b/plugins/linkedin/test/profile-read.test.js
index 3d49a5bc..1ed6eb92 100644
--- a/plugins/linkedin/test/profile-read.test.js
+++ b/plugins/linkedin/test/profile-read.test.js
@@ -53,20 +53,30 @@ describe('linkedin profile-read adapter', () => {
});
});
- it('does not require edit access when reading an explicit profile URL', async () => {
- const page = {
+ const workspacePage = (row, scrollRounds) => {
+ const rounds = [...scrollRounds];
+ const evaluated = [];
+ return {
+ evaluated,
goto: vi.fn(async () => {}),
wait: vi.fn(async () => {}),
autoScroll: vi.fn(async () => {}),
- evaluate: vi.fn()
- .mockResolvedValueOnce(false)
- .mockResolvedValueOnce({
- profile_url: 'https://www.linkedin.com/in/alice/',
- name: 'Alice',
- headline: 'Engineer',
- about: 'Builds products',
- }),
+ evaluate: vi.fn(async (script) => {
+ evaluated.push(script);
+ if (script.includes('authwall')) return false;
+ if (script.includes('scrollTop')) return rounds.shift() ?? { found: [], atEnd: true };
+ return row;
+ }),
};
+ };
+
+ it('does not require edit access when reading an explicit profile URL', async () => {
+ const page = workspacePage({
+ profile_url: 'https://www.linkedin.com/in/alice/',
+ name: 'Alice',
+ headline: 'Engineer',
+ about: 'Builds products',
+ }, [{ found: ['about', 'experience', 'education', 'featured'], atEnd: false }]);
await expect(command.func(page, { 'profile-url': 'https://www.linkedin.com/in/alice/' }))
.resolves.toMatchObject([{ name: 'Alice', about: 'Builds products' }]);
@@ -74,4 +84,28 @@ describe('linkedin profile-read adapter', () => {
expect(page.goto).toHaveBeenCalledWith('https://www.linkedin.com/in/alice/');
expect(page.goto.mock.calls.some(([url]) => String(url).includes('/edit/forms/'))).toBe(false);
});
+
+ it('scrolls the inner container until the lazy sections load before extracting', async () => {
+ const page = workspacePage({
+ profile_url: 'https://www.linkedin.com/in/alice/',
+ name: 'Alice',
+ experience: 'Engineer at Acme',
+ education: 'Example University',
+ }, [
+ { found: [], atEnd: false, container: '#workspace' },
+ { found: ['about'], atEnd: false, container: '#workspace' },
+ { found: ['about', 'experience', 'education', 'featured'], atEnd: false, container: '#workspace' },
+ ]);
+
+ await expect(command.func(page, { 'profile-url': 'https://www.linkedin.com/in/alice/' }))
+ .resolves.toMatchObject([{ experience: 'Engineer at Acme', education: 'Example University' }]);
+
+ // window autoScroll stays for older layouts, but cannot move main#workspace
+ expect(page.autoScroll).toHaveBeenCalledWith({ times: 4, delayMs: 700 });
+ const scrollScripts = page.evaluated.filter((script) => script.includes('scrollTop'));
+ expect(scrollScripts).toHaveLength(3);
+ expect(scrollScripts[0]).toContain("main#workspace");
+ // the extraction runs only after the scroll rounds
+ expect(page.evaluated.at(-1)).toContain('readSection');
+ });
});
diff --git a/plugins/linkedin/test/shared-scroll.test.js b/plugins/linkedin/test/shared-scroll.test.js
new file mode 100644
index 00000000..b512b811
--- /dev/null
+++ b/plugins/linkedin/test/shared-scroll.test.js
@@ -0,0 +1,137 @@
+import { describe, expect, it, vi } from 'vitest';
+import { JSDOM } from 'jsdom';
+import { buildSectionScrollScript, scrollToSections } from '../shared.js';
+
+/**
+ * Runs the generated script against a JSDOM document. JSDOM reports every
+ * layout metric as 0, so the fixture defines scrollHeight/clientHeight
+ * explicitly and a settable scrollTop, which is exactly the state the script
+ * reads to decide which element scrolls.
+ */
+function runScript(html, { headings, scrollHeight = 4000, clientHeight = 800, overflowY = 'auto' } = {}) {
+ const dom = new JSDOM(html, { url: 'https://www.linkedin.com/in/alice/', runScripts: 'outside-only' });
+ const { window } = dom;
+ const workspace = window.document.querySelector('#workspace');
+ if (workspace) {
+ Object.defineProperty(workspace, 'scrollHeight', { value: scrollHeight, configurable: true });
+ Object.defineProperty(workspace, 'clientHeight', { value: clientHeight, configurable: true });
+ let top = 0;
+ Object.defineProperty(workspace, 'scrollTop', {
+ get: () => top,
+ set: (value) => { top = value; },
+ configurable: true,
+ });
+ workspace.style.overflowY = overflowY;
+ }
+ Object.defineProperty(window.document.documentElement, 'scrollHeight', { value: scrollHeight, configurable: true });
+ window.innerHeight = clientHeight;
+ const scrolled = [];
+ window.scrollTo = (x, y) => { scrolled.push(y); window.scrollY = y; };
+ const result = window.eval(buildSectionScrollScript(headings ?? ['experience', 'education']));
+ return { result, window, workspace, scrolled };
+}
+
+describe('linkedin section scrolling', () => {
+ it('scrolls main#workspace when the window scroller cannot move', () => {
+ const { result, workspace, scrolled } = runScript(`
+
+
+
+ `);
+
+ expect(result.container).toBe('#workspace');
+ expect(workspace.scrollTop).toBe(800);
+ expect(result.atEnd).toBe(false);
+ expect(result.found).toEqual([]);
+ expect(scrolled).toEqual([]);
+ });
+
+ it('reports the requested headings once they are in the DOM', () => {
+ const { result } = runScript(`
+
+
+
+
+ `);
+
+ expect(result.found).toEqual(['experience', 'education']);
+ });
+
+ it('falls back to the window scroller on layouts without an inner container', () => {
+ const { result, scrolled } = runScript(`
+
+
+
+ `);
+
+ expect(result.container).toBe('window');
+ expect(scrolled).toEqual([800]);
+ expect(result.found).toEqual(['experience']);
+ });
+
+ it('reports the end of the container so callers stop scrolling', () => {
+ const { result } = runScript(`
+
+ `, { scrollHeight: 1000, clientHeight: 900 });
+
+ expect(result.container).toBe('#workspace');
+ expect(result.atEnd).toBe(true);
+ });
+});
+
+describe('scrollToSections', () => {
+ const page = (payloads) => ({
+ evaluate: vi.fn().mockImplementation(async () => payloads.shift() ?? { found: [], atEnd: true }),
+ wait: vi.fn().mockResolvedValue(undefined),
+ });
+
+ it('keeps scrolling until every requested section has loaded', async () => {
+ const target = page([
+ { found: [], atEnd: false, container: '#workspace' },
+ { found: ['experience'], atEnd: false, container: '#workspace' },
+ { found: ['experience', 'education'], atEnd: false, container: '#workspace' },
+ ]);
+
+ await expect(scrollToSections(target, ['experience', 'education']))
+ .resolves.toMatchObject({ found: ['experience', 'education'] });
+ expect(target.evaluate).toHaveBeenCalledTimes(3);
+ });
+
+ it('stops once the container is stably at its end and no section appeared', async () => {
+ const target = page([
+ { found: [], atEnd: false, container: '#workspace' },
+ { found: [], atEnd: true, container: '#workspace' },
+ { found: [], atEnd: true, container: '#workspace' },
+ ]);
+
+ await expect(scrollToSections(target, ['experience'])).resolves.toMatchObject({ atEnd: true });
+ expect(target.evaluate).toHaveBeenCalledTimes(3);
+ });
+
+ it('keeps scrolling past an atEnd round that lazy-loaded a new section', async () => {
+ const target = page([
+ { found: [], atEnd: true, container: '#workspace' },
+ { found: ['experience'], atEnd: true, container: '#workspace' },
+ { found: ['experience', 'education'], atEnd: true, container: '#workspace' },
+ ]);
+
+ await expect(scrollToSections(target, ['experience', 'education']))
+ .resolves.toMatchObject({ found: ['experience', 'education'] });
+ expect(target.evaluate).toHaveBeenCalledTimes(3);
+ });
+
+ it('gives up after the round budget instead of scrolling forever', async () => {
+ const target = page([]);
+ target.evaluate.mockResolvedValue({ found: [], atEnd: false, container: '#workspace' });
+
+ await scrollToSections(target, ['experience'], { rounds: 3 });
+
+ expect(target.evaluate).toHaveBeenCalledTimes(3);
+ });
+
+ it('never fails the command when the page cannot be evaluated', async () => {
+ const target = { evaluate: vi.fn().mockRejectedValue(new Error('page closed')), wait: vi.fn() };
+
+ await expect(scrollToSections(target, ['experience'])).resolves.toMatchObject({ found: [] });
+ });
+});