From a1563cf5af13474824fbc311c631558c01a24688 Mon Sep 17 00:00:00 2001 From: developmentconexus Date: Wed, 11 Nov 2026 19:11:20 -0300 Subject: [PATCH 1/8] =?UTF-8?q?feat(connectors):=20o=20gravador=20de=20con?= =?UTF-8?q?ectores:=20a=20leitura=20declarada=20sai=20pelo=20mesmo=20pedid?= =?UTF-8?q?o=20do=20Hub,=20o=20corpo=20cru=20fica=20fora=20do=20reposit?= =?UTF-8?q?=C3=B3rio=20e=20a=20troca=20por=20valores=20inventados=20grava?= =?UTF-8?q?=20a=20grava=C3=A7=C3=A3o?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package.json | 1 + scripts/record-connector-calls.ts | 103 +++++++++++++++++++ scripts/record-connector-swap.ts | 129 ++++++++++++++++++++++++ scripts/record-connector.test.ts | 159 ++++++++++++++++++++++++++++++ scripts/record-connector.ts | 84 ++++++++++++++++ 5 files changed, 476 insertions(+) create mode 100644 scripts/record-connector-calls.ts create mode 100644 scripts/record-connector-swap.ts create mode 100644 scripts/record-connector.test.ts create mode 100644 scripts/record-connector.ts diff --git a/package.json b/package.json index 61bc3fb4..d679a248 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "removals": "node scripts/removals.ts", "record": "node scripts/record.ts", "record-provider": "node scripts/record-provider.ts", + "record-connector": "node scripts/record-connector.ts", "openapi": "node scripts/emit-openapi.ts", "radius": "node scripts/check-radius.ts", "boundaries": "node scripts/check-boundaries.ts", diff --git a/scripts/record-connector-calls.ts b/scripts/record-connector-calls.ts new file mode 100644 index 00000000..0702b655 --- /dev/null +++ b/scripts/record-connector-calls.ts @@ -0,0 +1,103 @@ +// The first step of `scripts/record-connector.ts`: one declared read, built and sent as the Hub +// sends it (`requestOf`, then `toRequest`, as `sentFor` does in the executor) and its bytes read by +// the Hub's own `bodyOf`, kept before `operation.answer` reads them, as Nango's interceptor keeps +// `response.data` (dryrun.service.ts:468-475). The credential is a parameter and never leaves it. +import { isAbsolute, relative, resolve } from 'node:path' +import type { Connector } from '@conexus/connectors' +import { AccountRef, requestOf, toRequest } from '@conexus/connectors' +import { z } from 'zod' +import { bodyOf } from '../apps/server/src/hub/connectors/wire.ts' + +/** The reads the card names: orders by changed date, the listing scan's two pages and the questions scan. */ +export const CASES = [ + 'pedidos-alterados', + 'anuncios-varredura-1', + 'anuncios-varredura-2', + 'perguntas-varredura', +] as const +export type Case = (typeof CASES)[number] + +/** What the raw file in the private folder keeps: the route as sent (no credential), the status and the body as the service sent it. */ +export const Raw = z.object({ + case: z.enum(CASES), + connector: z.string(), + method: z.string(), + path: z.string(), + query: z.record(z.string(), z.string()), + status: z.number().int(), + body: z.unknown(), + recordedAt: z.iso.date(), +}) +export type Raw = z.infer + +const PAGE_1 = 'anuncios-varredura-1' + +/** The operation of a case and its input; the second listing page carries the first page's `scroll_id`. */ +export function inputOf( + name: Case, + now: Date, + first: unknown, +): { operation: string; input: unknown } { + if (name === 'pedidos-alterados') { + const week = new Date(now.getTime() - 7 * 24 * 60 * 60 * 1000) + return { + operation: 'pedidos', + input: { changedFrom: week.toISOString(), offset: 0, limit: 50 }, + } + } + if (name === PAGE_1) return { operation: 'anuncios', input: { scan: true, limit: 100 } } + if (name === 'perguntas-varredura') + return { operation: 'perguntas', input: { scan: true, limit: 50 } } + const scroll = z.object({ scroll_id: z.string().min(1) }).safeParse(first) + if (!scroll.success) + throw new Error('The first page carries no scroll_id; the second page cannot be asked.') + return { operation: 'anuncios', input: { scrollId: scroll.data.scroll_id, limit: 100 } } +} + +/** A folder outside the repository: a raw body inside it could be committed. */ +export function assertOutside(out: string, root: string): string { + const path = resolve(out) + const inside = relative(root, path) + if (!inside.startsWith('..') && !isAbsolute(inside)) + throw new Error('--out must be a folder outside the repository.') + return path +} + +/** + * Sends one case and returns what arrived. A refusal or an unreadable body is thrown by status + * only, nothing is returned, and the credential and the account are never in the message. + */ +export async function recorded( + a: Readonly<{ + fetch: typeof fetch + connector: Connector + name: Case + account: string + token: string + now: Date + first: unknown + }>, +): Promise { + const { operation: id, input } = inputOf(a.name, a.now, a.first) + const operation = a.connector.operations.find((candidate) => candidate.id === id) + if (operation === undefined) throw new Error(`${a.connector.id} declares no ${id}`) + const request = requestOf(operation, input, { ref: AccountRef.parse(a.account) }, a.now) + const response = await a.fetch(toRequest(a.connector, request, a.token), { + signal: AbortSignal.timeout(60_000), + }) + const bytes = await bodyOf(response) + if (bytes === null || !response.ok) + throw new Error(`The service answered ${response.status}; nothing is saved.`) + const text = Buffer.from(bytes).toString('utf8') + if (text.includes(a.token)) throw new Error('The body carries the credential; nothing is saved.') + return { + case: a.name, + connector: a.connector.id, + method: request.method, + path: request.path, + query: request.query, + status: response.status, + body: z.unknown().parse(JSON.parse(text)), + recordedAt: a.now.toISOString().slice(0, 10), + } +} diff --git a/scripts/record-connector-swap.ts b/scripts/record-connector-swap.ts new file mode 100644 index 00000000..dc36434b --- /dev/null +++ b/scripts/record-connector-swap.ts @@ -0,0 +1,129 @@ +// Defect: a connector double answers in a shape the real service never sends (testing T24, T29). +// The second step of `scripts/record-connector.ts`: a raw body kept in the operator's private folder +// becomes a recording that may enter the repository, every value replaced by an invented one of the +// same shape, the same real value always by the same invented one (security S40). Nango keeps the +// raw body and cleans nothing; the swap is our rule beyond theirs. +import { LEAK } from '../apps/server/src/platform/test-recordings.ts' + +/** A place in the body, as dotted keys (`results.0.buyer.nickname`), the way the kit names a path. */ +type Path = readonly string[] + +const ANCHOR = Date.parse('2026-09-15T12:00:00.000Z') +const INSTANT = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?(Z|[+-]\d{2}:\d{2})$/ + +/** Keys whose number is an identity or an amount: replaced. Paging counts and quantities stay. */ +const NUMBER_KEY = /(^|_)id$|^id$|seller|buyer|number|price|amount|cost|fee|tax/i +/** Keys whose text is a person's, a seller's or a listing's own words: always replaced, whatever it looks like. */ +const TEXT_KEY = + /name|nick|title|text|email|phone|address|street|city|zip|permalink|thumbnail|picture|sku|seller_custom_field|document|number|comment|description|identification/i +/** Keys whose text names a state of the service, not a person or a thing: kept when it looks like a code. */ +const CODE = /^[a-z][a-z0-9_]{0,40}$/ +/** Keys whose text is an identity: replaced by one of the same shape. */ +const ID_KEY = /(^|_)id$|^id$|scroll|seller|buyer/i + +export type Swapped = Readonly<{ body: unknown; replaced: readonly string[]; path: string }> + +/** The seller's own value in the route: the account the connection is for, so the path and the body agree. */ +type Seed = Readonly<{ account: string; path: string }> + +class Invented { + private readonly known = new Map() + private next = 1 + readonly replaced = new Set() + + /** The invented value for a real one: the same real value gives the same invented one. */ + of(value: string | number, key: string, at: Path): string | number { + this.replaced.add(at.filter((part) => !/^\d+$/.test(part)).join('.')) + const id = String(value) + const kept = this.known.get(id) + if (kept !== undefined) return typeof value === 'number' ? Number(kept) : String(kept) + const made = this.make(value, key) + this.known.set(id, made) + return made + } + + private make(value: string | number, key: string): string | number { + const n = this.next++ + if (typeof value === 'number') { + if (!Number.isInteger(value)) return Math.round((10 + n * 7.35) * 100) / 100 + const digits = String(Math.abs(value)).length + return 2 * 10 ** (digits - 1) + n + } + if (ID_KEY.test(key)) { + const prefix = value.replace(/[0-9].*$/, '').replace(/[^A-Za-z]/g, '') + const width = Math.max(value.length - prefix.length, 6) + return `${prefix.toUpperCase()}${String(n).padStart(width, '0')}` + } + return `Texto inventado ${n}` + } +} + +/** Every instant moved by one shift so the latest one is the anchor: the order and the gaps stay. */ +function instantsOf(value: unknown, found: number[] = []): number[] { + if (typeof value === 'string' && INSTANT.test(value)) found.push(Date.parse(value)) + else if (Array.isArray(value)) for (const item of value) instantsOf(item, found) + else if (typeof value === 'object' && value !== null) + for (const item of Object.values(value)) instantsOf(item, found) + return found +} + +/** The instant moved by `by`, written as it was: the same offset and the same fraction of a second. */ +function shifted(value: string, by: number): string { + const zone = /(Z|[+-]\d{2}:\d{2})$/.exec(value)?.[0] ?? 'Z' + const sign = zone.startsWith('-') ? -1 : 1 + const minutes = + zone === 'Z' ? 0 : sign * (Number(zone.slice(1, 3)) * 60 + Number(zone.slice(4, 6))) + const wall = new Date(Date.parse(value) + by + minutes * 60_000).toISOString() + return `${wall.slice(0, /\.\d+/.test(value) ? 23 : 19)}${zone}` +} + +function walk( + value: unknown, + key: string, + at: Path, + a: Readonly<{ invented: Invented; by: number }>, +): unknown { + if (Array.isArray(value)) return value.map((item, i) => walk(item, key, [...at, String(i)], a)) + if (typeof value === 'object' && value !== null) + return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, walk(v, k, [...at, k], a)])) + if (typeof value === 'number') return NUMBER_KEY.test(key) ? a.invented.of(value, key, at) : value + if (typeof value !== 'string') return value + if (INSTANT.test(value)) { + a.invented.replaced.add(at.filter((part) => !/^\d+$/.test(part)).join('.')) + return shifted(value, a.by) + } + if (!TEXT_KEY.test(key) && !ID_KEY.test(key) && CODE.test(value)) return value + return a.invented.of(value, key, at) +} + +/** What of the real body still stands in the swapped one: text with a digit run or a space that the swap kept. */ +function survivors(raw: unknown, swapped: unknown): string[] { + const real = new Set(stringsOf(raw).filter((text) => /\d{5,}| /.test(text))) + return stringsOf(swapped).filter((text) => real.has(text)) +} + +function stringsOf(value: unknown): string[] { + if (typeof value === 'string') return [value] + if (typeof value !== 'object' || value === null) return [] + return Object.values(value).flatMap(stringsOf) +} + +/** + * The raw body with every identity, name, text, amount and date replaced (dates moved together), + * and the route's path with the account replaced. Throws when a real text survives or an address or + * a credential's prefix is left (`LEAK`): nothing may be saved then. + */ +export function swapped(raw: Readonly<{ body: unknown } & Seed>): Swapped { + const invented = new Invented() + const latest = Math.max(...instantsOf(raw.body), ANCHOR) + const by = ANCHOR - latest + const body = walk(raw.body, '', [], { invented, by }) + const account = String(invented.of(Number(raw.account), 'seller', ['seller'])) + const path = raw.path.replaceAll(raw.account, account) + const left = survivors(raw.body, body) + if (left.length > 0) + throw new Error(`The swap left ${left.length} real text(s); nothing is saved.`) + if (LEAK.test(JSON.stringify(body)) || LEAK.test(path)) + throw new Error('The swapped body still carries a credential or an address; nothing is saved.') + return { body, path, replaced: [...invented.replaced].sort() } +} diff --git a/scripts/record-connector.test.ts b/scripts/record-connector.test.ts new file mode 100644 index 00000000..ea5b8092 --- /dev/null +++ b/scripts/record-connector.test.ts @@ -0,0 +1,159 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { CONNECTORS } from '@conexus/connectors' +import { z } from 'zod' +import { assertOutside, recorded } from './record-connector-calls.ts' +import { swapped } from './record-connector-swap.ts' + +const ML = CONNECTORS.find((connector) => connector.id === 'mercado-livre') +if (ML === undefined) throw new Error('No Mercado Livre') +const NOW = new Date('2026-11-11T12:00:00.000Z') +// Invented bytes in the shape of the service's bodies; the real ones never enter a test. +const SELLER = '987654321' +const TOKEN = 'APP_USR-fake-token-0001' + +function answering(bodies: unknown[]) { + const seen: Request[] = [] + const fake: typeof fetch = async (input) => { + seen.push(input instanceof Request ? input : new Request(input)) + return new Response(JSON.stringify(bodies.shift()), { + status: 200, + headers: { 'content-type': 'application/json' }, + }) + } + return { fake, seen } +} + +const ARGS = { connector: ML, account: SELLER, token: TOKEN, now: NOW, first: null } + +test('o gravador monta o pedido pela declaração da operação e guarda o corpo como chegou', async () => { + const body = { paging: { total: 1, offset: 0, limit: 50 }, results: [{ id: 1 }] } + const { fake, seen } = answering([body]) + const raw = await recorded({ ...ARGS, fetch: fake, name: 'pedidos-alterados' }) + const url = new URL(seen[0]?.url ?? '') + assert.equal(url.pathname, '/orders/search') + assert.equal(url.searchParams.get('seller'), SELLER) + assert.equal(url.searchParams.get('order.date_last_updated.from'), '2026-11-04T12:00:00.000Z') + assert.equal(seen[0]?.headers.get('authorization'), `Bearer ${TOKEN}`) + assert.deepEqual(raw.body, body) + assert.equal(JSON.stringify(raw).includes(TOKEN), false) +}) + +test('a segunda página da varredura manda o scroll_id da primeira', async () => { + const { fake, seen } = answering([{ results: ['MLB1'] }]) + await recorded({ + ...ARGS, + fetch: fake, + name: 'anuncios-varredura-2', + first: { scroll_id: 'SCROLL0001', results: ['MLB1'] }, + }) + const url = new URL(seen[0]?.url ?? '') + assert.equal(url.searchParams.get('scroll_id'), 'SCROLL0001') + assert.equal(url.searchParams.get('search_type'), 'scan') +}) + +test('sem scroll_id na primeira página a segunda não é pedida', async () => { + const { fake, seen } = answering([]) + await assert.rejects( + recorded({ ...ARGS, fetch: fake, name: 'anuncios-varredura-2', first: { results: [] } }), + /no scroll_id/, + ) + assert.equal(seen.length, 0) +}) + +test('uma recusa do serviço não grava nada e a mensagem não traz a credencial', async () => { + const refuse: typeof fetch = async () => new Response('{"message":"x"}', { status: 403 }) + await assert.rejects( + recorded({ ...ARGS, fetch: refuse, name: 'perguntas-varredura' }), + (error: Error) => error.message.includes('403') && !error.message.includes(TOKEN), + ) +}) + +test('um corpo que traz a credencial não é guardado', async () => { + const { fake } = answering([{ echo: TOKEN }]) + await assert.rejects( + recorded({ ...ARGS, fetch: fake, name: 'perguntas-varredura' }), + /carries the credential/, + ) +}) + +test('a pasta de saída dentro do repositório é recusada', () => { + assert.throws(() => assertOutside('/repo/data/raw', '/repo'), /outside the repository/) + assert.throws(() => assertOutside('/repo', '/repo'), /outside the repository/) + assert.equal(assertOutside('/elsewhere/raw', '/repo'), '/elsewhere/raw') +}) + +const Swapped = z.object({ + paging: z.object({ total: z.number() }), + results: z + .array( + z.object({ + id: z.number(), + status: z.string(), + date_created: z.string().optional(), + date_last_updated: z.string().optional(), + seller: z.object({ id: z.number() }).optional(), + }), + ) + .min(1), +}) + +const REAL = { + paging: { total: 1300, limit: 50 }, + scroll_id: 'AbCdEf0123456789xyzREALscroll', + results: [ + { + id: 2000007654321, + status: 'paid', + date_created: '2026-11-01T08:30:15.000-03:00', + date_last_updated: '2026-11-02T10:00:00.000-03:00', + total_amount: 189.9, + seller: { id: 987654321, nickname: 'LOJAREALDAMARIA' }, + item: { id: 'MLB3456789012', title: 'Camiseta Maria Silva', unit_price: 94.95 }, + buyer: { id: 55555555, nickname: 'COMPRADORREAL', email: 'real@exemplo.test' }, + }, + { id: 2000007654322, status: 'cancelled', total_amount: 10 }, + ], +} + +test('a troca muda ids, nomes, preços e datas, e guarda a forma', () => { + const done = swapped({ body: REAL, account: SELLER, path: `/users/${SELLER}/items/search` }) + const text = JSON.stringify(done.body) + for (const real of [ + 'AbCdEf', + 'LOJAREAL', + 'Maria Silva', + 'COMPRADORREAL', + '2000007654321', + '189.9', + ]) + assert.equal(text.includes(real), false, real) + assert.equal(done.path.includes(SELLER), false) + const body = Swapped.parse(done.body) + const first = body.results[0] + assert.equal(first?.status, 'paid') + assert.equal(body.paging.total, 1300) + assert.match(first?.date_created ?? '', /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}-03:00$/) + assert.equal(String(first?.id).length, '2000007654321'.length) + assert.ok(done.replaced.includes('results.buyer.nickname')) + assert.ok(done.replaced.includes('scroll_id')) +}) + +test('o mesmo valor real vira o mesmo inventado, e a ordem das datas fica', () => { + const done = swapped({ body: REAL, account: SELLER, path: `/users/${SELLER}/x` }) + const body = Swapped.parse(done.body) + assert.equal(String(body.results[0]?.seller?.id), done.path.split('/')[2]) + const [created, updated] = [body.results[0]?.date_created, body.results[0]?.date_last_updated] + assert.ok(Date.parse(created ?? '') < Date.parse(updated ?? '')) + assert.equal( + Date.parse(updated ?? '') - Date.parse(created ?? ''), + (25 * 60 + 29) * 60_000 + 45_000, + ) +}) + +test('um endereço ou prefixo de credencial que sobra recusa a gravação', () => { + assert.throws( + () => swapped({ body: { 'maria@exemplo.test': 'x' }, account: SELLER, path: '/x' }), + /nothing is saved/, + ) +}) diff --git a/scripts/record-connector.ts b/scripts/record-connector.ts new file mode 100644 index 00000000..d0f8b1bc --- /dev/null +++ b/scripts/record-connector.ts @@ -0,0 +1,84 @@ +// Defect: a connector double answers in a shape the real service never sends, so a test passes on +// a part the connector wants and the real body breaks it (testing T24, T29). +// `npm run record-connector -- --connector mercado-livre --case --out ` sends one +// of the card's four reads (`pedidos-alterados`, `anuncios-varredura-1`, `anuncios-varredura-2`, +// `perguntas-varredura`) as the Hub builds and sends it, and keeps the body as it arrived in +// `/.raw.json`; the folder must be outside the repository, since the body is real. +// The second page of the scan reads the first page's `scroll_id` from the same folder. +// `... --case --out --swap` replaces every value of that raw body by an invented +// one of the same shape and saves the recording in `packages/connectors/src//recordings/`, +// with the service, the route, the date and the paths it replaced beside it. +// The credential comes from the environment only and never reaches a file or the output (security +// S40): CONEXUS_RECORD_ML_TOKEN (a seller's access token) and CONEXUS_RECORD_ML_SELLER (the +// seller's id, the account the read is for). +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import { parseArgs } from 'node:util' +import { CONNECTORS } from '@conexus/connectors' +import { z } from 'zod' +import { assertOutside, CASES, Raw, recorded } from './record-connector-calls.ts' +import { swapped } from './record-connector-swap.ts' + +const ROOT = join(import.meta.dirname, '..') +const Args = z.object({ + connector: z.literal('mercado-livre'), + case: z.enum(CASES), + out: z.string().min(1), + swap: z.boolean().default(false), +}) +const Seller = z.object({ CONEXUS_RECORD_ML_SELLER: z.string().min(1) }) +const Env = Seller.extend({ CONEXUS_RECORD_ML_TOKEN: z.string().min(1) }) + +const { values } = parseArgs({ + options: { + connector: { type: 'string' }, + case: { type: 'string' }, + out: { type: 'string' }, + swap: { type: 'boolean' }, + }, +}) +const args = Args.parse(values) +const out = assertOutside(args.out, ROOT) +const connector = CONNECTORS.find((candidate) => candidate.id === args.connector) +if (connector === undefined) throw new Error(`No connector ${args.connector}`) +function rawPath(name: string) { + return join(out, `${name}.raw.json`) +} + +if (args.swap) { + const raw = Raw.parse(JSON.parse(readFileSync(rawPath(args.case), 'utf8'))) + const { CONEXUS_RECORD_ML_SELLER: account } = Seller.parse(process.env) + const done = swapped({ body: raw.body, account, path: raw.path }) + const folder = join(ROOT, 'packages/connectors/src', connector.id, 'recordings') + mkdirSync(folder, { recursive: true }) + const recording = { + service: connector.name, + connector: connector.id, + case: raw.case, + route: { method: raw.method, path: done.path, query: Object.keys(raw.query).sort() }, + recordedAt: raw.recordedAt, + status: raw.status, + replaced: done.replaced, + body: done.body, + } + writeFileSync(join(folder, `${raw.case}.json`), `${JSON.stringify(recording, null, 2)}\n`) + console.log(`Saved ${raw.case}.json with ${done.replaced.length} replaced paths.`) +} else { + const env = Env.parse(process.env) + const first = + args.case === 'anuncios-varredura-2' + ? Raw.parse(JSON.parse(readFileSync(rawPath('anuncios-varredura-1'), 'utf8'))).body + : null + const raw = await recorded({ + fetch, + connector, + name: args.case, + account: env.CONEXUS_RECORD_ML_SELLER, + token: env.CONEXUS_RECORD_ML_TOKEN, + now: new Date(), + first, + }) + mkdirSync(out, { recursive: true }) + writeFileSync(rawPath(args.case), JSON.stringify(raw, null, 2)) + console.log(`Kept ${args.case}.raw.json (status ${raw.status}) in the folder.`) +} From 7cf9dec1126bf44945059b1c4b85dd6f8f111bd8 Mon Sep 17 00:00:00 2001 From: developmentconexus Date: Wed, 11 Nov 2026 19:12:41 -0300 Subject: [PATCH 2/8] =?UTF-8?q?fix(connectors):=20a=20skill=20e=20os=20res?= =?UTF-8?q?umos=20do=20Mercado=20Livre=20dizem=20a=20varredura=20e=20o=20f?= =?UTF-8?q?iltro=20de=20altera=C3=A7=C3=A3o,=20e=20a=20varredura=20recusa?= =?UTF-8?q?=20offset?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/mercado-livre/mercado-livre.test.ts | 10 ++++++ .../src/mercado-livre/operations.test.ts | 2 ++ .../src/mercado-livre/operations/anuncios.ts | 21 ++++++----- .../src/mercado-livre/operations/perguntas.ts | 36 ++++++++++--------- .../src/mercado-livre/operations/shared.ts | 10 ++++++ .../skill/references/anuncios.md | 2 +- .../skill/references/perguntas.md | 2 +- .../mercado-livre/skill/references/vendas.md | 2 +- 8 files changed, 56 insertions(+), 29 deletions(-) diff --git a/packages/connectors/src/mercado-livre/mercado-livre.test.ts b/packages/connectors/src/mercado-livre/mercado-livre.test.ts index 46484830..16d1b46f 100644 --- a/packages/connectors/src/mercado-livre/mercado-livre.test.ts +++ b/packages/connectors/src/mercado-livre/mercado-livre.test.ts @@ -9,6 +9,7 @@ import { authorizeUrl, codeRequest, refreshRequest } from '../kernel/oauth.ts' import { AccountRef, requestOf } from '../kernel/operation.ts' import { toRequest } from '../kernel/request.ts' import { Rows } from '../kernel/rows.ts' +import { readSkill } from '../kernel/skill.ts' import { mercadoLivre } from './definition.ts' import { mercadoLivreFake } from './fake.ts' import { docsOf } from './operations/shared.ts' @@ -145,6 +146,15 @@ test('só anuncios e perguntas nomeiam o cursor de varredura, no resumo, na entr } }) +test('a skill diz o mesmo que os resumos: a varredura passa de mil e o filtro de alteração existe', () => { + const { references } = readSkill(new URL('./skill/', import.meta.url)) + for (const name of ['anuncios.md', 'perguntas.md']) + assert.match(references[name] ?? '', /`scan: true`.*`scrollId`/s, name) + assert.match(references['vendas.md'] ?? '', /`changedFrom`/) + for (const [name, text] of Object.entries(references)) + assert.doesNotMatch(text, /no way past 1000/, name) +}) + test('o conector declara que o Mercado Livre conta o limite pelo aplicativo', () => { assert.deepEqual(mercadoLivre.callLimit, { countedBy: 'APP' }) }) diff --git a/packages/connectors/src/mercado-livre/operations.test.ts b/packages/connectors/src/mercado-livre/operations.test.ts index 3f3c39e9..c287ec0f 100644 --- a/packages/connectors/src/mercado-livre/operations.test.ts +++ b/packages/connectors/src/mercado-livre/operations.test.ts @@ -286,6 +286,8 @@ test('a varredura: a primeira página manda scan, as seguintes o scroll_id', () `${id}: por deslocamento, como hoje`, ) assert.equal(read.input.safeParse({ scan: false, limit: 50 }).success, false, id) + for (const extra of [{ scan: true }, { scrollId: 'abc' }]) + assert.equal(read.input.safeParse({ ...extra, offset: 0, limit: 50 }).success, false, id) assert.equal(read.input.safeParse({ scrollId: '', limit: 50 }).success, false, id) } }) diff --git a/packages/connectors/src/mercado-livre/operations/anuncios.ts b/packages/connectors/src/mercado-livre/operations/anuncios.ts index f108e4f6..e746feee 100644 --- a/packages/connectors/src/mercado-livre/operations/anuncios.ts +++ b/packages/connectors/src/mercado-livre/operations/anuncios.ts @@ -8,6 +8,7 @@ import { PathId, readOf, Scan, + scanAlone, searchTypeOf, sellerOf, } from './shared.ts' @@ -19,15 +20,17 @@ export const anuncios = readOf({ summary: 'Lista os ids dos anúncios do vendedor, filtrando por status ou SKU, uma página por chamada (até 100). Com `scan` a busca é uma varredura que passa de mil: a primeira página manda `scan`, e a resposta traz o `seguinte` (o `scrollId` da próxima, que vale 5 minutos).', label: 'Anúncios', - input: z.object({ - status: z - .enum(['active', 'paused', 'closed', 'under_review', 'inactive', 'pending']) - .optional(), - sku: z.string().min(1).max(100).optional(), - offset: Offset.optional(), - ...Scan, - limit: Limit(100), - }), + input: z + .object({ + status: z + .enum(['active', 'paused', 'closed', 'under_review', 'inactive', 'pending']) + .optional(), + sku: z.string().min(1).max(100).optional(), + offset: Offset.optional(), + ...Scan, + limit: Limit(100), + }) + .refine(scanAlone.check, scanAlone.issue), list: 'results', probe: () => ({ offset: 0, limit: 5 }), request: (input, account) => diff --git a/packages/connectors/src/mercado-livre/operations/perguntas.ts b/packages/connectors/src/mercado-livre/operations/perguntas.ts index 570d5469..2d55da11 100644 --- a/packages/connectors/src/mercado-livre/operations/perguntas.ts +++ b/packages/connectors/src/mercado-livre/operations/perguntas.ts @@ -1,6 +1,6 @@ import { z } from 'zod' import { get } from '../../kernel/operation.ts' -import { Limit, Offset, readOf, Scan, searchTypeOf, sellerOf } from './shared.ts' +import { Limit, Offset, readOf, Scan, scanAlone, searchTypeOf, sellerOf } from './shared.ts' export const perguntas = readOf({ id: 'perguntas', @@ -9,22 +9,24 @@ export const perguntas = readOf({ summary: 'Lista as perguntas feitas nos anúncios do vendedor, com a resposta quando há, uma página por chamada (até 50). As sem resposta aparecem por 7 meses. Com `scan` a busca é uma varredura que passa de mil: a primeira página manda `scan`, e a resposta traz o `seguinte` (o `scrollId` da próxima, que vale 5 minutos).', label: 'Perguntas', - input: z.object({ - status: z - .enum([ - 'UNANSWERED', - 'ANSWERED', - 'CLOSED_UNANSWERED', - 'UNDER_REVIEW', - 'BANNED', - 'DELETED', - 'DISABLED', - ]) - .optional(), - offset: Offset.optional(), - ...Scan, - limit: Limit(50), - }), + input: z + .object({ + status: z + .enum([ + 'UNANSWERED', + 'ANSWERED', + 'CLOSED_UNANSWERED', + 'UNDER_REVIEW', + 'BANNED', + 'DELETED', + 'DISABLED', + ]) + .optional(), + offset: Offset.optional(), + ...Scan, + limit: Limit(50), + }) + .refine(scanAlone.check, scanAlone.issue), list: 'questions', omit: ['from'], // the asker: the question route adds their name and contact probe: () => ({ offset: 0, limit: 5 }), diff --git a/packages/connectors/src/mercado-livre/operations/shared.ts b/packages/connectors/src/mercado-livre/operations/shared.ts index c4c54bdd..094df3d4 100644 --- a/packages/connectors/src/mercado-livre/operations/shared.ts +++ b/packages/connectors/src/mercado-livre/operations/shared.ts @@ -24,6 +24,16 @@ export function searchTypeOf(input: Readonly<{ scan?: true; scrollId?: string }> return input.scan === true || input.scrollId !== undefined ? 'scan' : undefined } +/** A scan pages by its cursor, not by `offset`: the two together are refused where the input is read, so no request carries both. */ +export const scanAlone = { + check: (input: Readonly<{ scan?: true; scrollId?: string; offset?: number }>) => + searchTypeOf(input) === undefined || input.offset === undefined, + issue: { + message: 'A scan pages by scrollId; send no offset with scan or scrollId', + path: ['offset'], + }, +} + /** The seller's id in a path: the connection's account, which the Hub parses again when it opens the connection. */ export function sellerOf(account: Account): string { if (account.ref === null) throw new Error('Mercado Livre read without an account') diff --git a/packages/connectors/src/mercado-livre/skill/references/anuncios.md b/packages/connectors/src/mercado-livre/skill/references/anuncios.md index 0784b0f1..6ca683d5 100644 --- a/packages/connectors/src/mercado-livre/skill/references/anuncios.md +++ b/packages/connectors/src/mercado-livre/skill/references/anuncios.md @@ -1,6 +1,6 @@ # Anúncios -- `anuncios`: the ids of the seller's listings, filtered by `status` or `sku`. Up to 100 per page (`offset`, `limit`). The search reads at most 1000 listings (Mercado Livre's offset cap): a seller with more narrows it with `status` or `sku`; there is no way past 1000. +- `anuncios`: the ids of the seller's listings, filtered by `status` or `sku`. Up to 100 per page (`offset`, `limit`). `offset` reads at most 1000 listings (Mercado Livre's offset cap); past 1000 the search is a scan: the first page sends `scan: true` (never with `offset`), the answer's `seguinte` is the `scrollId` of the next page, which is valid for 5 minutes, and the page with no rows is the end. - `anunciosDetalhe`: up to 20 listings per call by id (title, price, stock, sold, status, category, listing type). Each item has its own `code`: a 404 there is that id's, not the call's. - `descricao`: the plain-text description of one listing. - `estoque`: the stock of one user product by warehouse. Limit of 100 calls per minute. diff --git a/packages/connectors/src/mercado-livre/skill/references/perguntas.md b/packages/connectors/src/mercado-livre/skill/references/perguntas.md index 8c056edb..3ce91316 100644 --- a/packages/connectors/src/mercado-livre/skill/references/perguntas.md +++ b/packages/connectors/src/mercado-livre/skill/references/perguntas.md @@ -1,4 +1,4 @@ # Perguntas -- `perguntas`: questions on the seller's listings with the answer when there is one, up to 50 per page, filtered by `status`. Unanswered questions stay visible for 7 months. The asker is not in the answer. The search reads at most 1000 questions (Mercado Livre's offset cap): with more, narrow it with `status`; there is no way past 1000. +- `perguntas`: questions on the seller's listings with the answer when there is one, up to 50 per page, filtered by `status`. Unanswered questions stay visible for 7 months. The asker is not in the answer. `offset` reads at most 1000 questions (Mercado Livre's offset cap); past 1000 the search is a scan: the first page sends `scan: true` (never with `offset`), the answer's `seguinte` is the `scrollId` of the next page, which is valid for 5 minutes, and the page with no rows is the end. - `tempoDeResposta`: the seller's average answer time, in minutes. diff --git a/packages/connectors/src/mercado-livre/skill/references/vendas.md b/packages/connectors/src/mercado-livre/skill/references/vendas.md index 03c834ed..25dbe3cf 100644 --- a/packages/connectors/src/mercado-livre/skill/references/vendas.md +++ b/packages/connectors/src/mercado-livre/skill/references/vendas.md @@ -1,6 +1,6 @@ # Vendas -- `pedidos`: the seller's orders, up to 50 per page. Orders are kept for 12 months; filter by `status` and by `dateFrom`/`dateTo` (ISO 8601 with offset) and sort with `date_asc` or `date_desc`. An order can come partial (`206`): it is read, and `parcial: true` says some of its fields are missing. +- `pedidos`: the seller's orders, up to 50 per page. Orders are kept for 12 months; filter by `status` and by `dateFrom`/`dateTo` (ISO 8601 with offset) sort with `date_asc` or `date_desc`, and read only what changed since an instant with `changedFrom`. An order can come partial (`206`): it is read, and `parcial: true` says some of its fields are missing. - `pedido`: one order by id, with its items, its `pack_id` and its shipping id. - `pacote`: one pack (several orders of one buyer shipped together). Only orders with a `pack_id` have one. From 9df1bbce8bf9488e04269d6ef1ee854973abfc30 Mon Sep 17 00:00:00 2001 From: developmentconexus Date: Wed, 11 Nov 2026 20:57:51 -0300 Subject: [PATCH 3/8] =?UTF-8?q?feat(connectors):=20as=20cinco=20leituras?= =?UTF-8?q?=20de=20lista=20do=20Mercado=20Livre=20declaram=20como=20pagina?= =?UTF-8?q?m,=20e=20o=20dubl=C3=AA=20pagina=20(U2b,=20sem=20grava=C3=A7?= =?UTF-8?q?=C3=B5es)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit O Sync de qualquer conector, U2, a parte de código, pela decisão do firstmate de 2026-11-11 (a U3 prova o motor pelo dublê do Mercado Livre): pedidos declara OFFSET (offset, limit, 50, sem teto) e o filtro do que mudou (changedFrom, date_last_updated); anuncios e perguntas declaram CURSOR (scrollId, limit, 100 e 50, a primeira página com scan, 5 minutos); reclamacoes e periodosDeFaturamento, OFFSET; mensagensDoPacote não declara. O dublê tem 120 pedidos com a data de alteração e o filtro, 1.300 anúncios e a varredura com scroll_id em anúncios e perguntas. A skill deixa de dizer que não há como passar de mil. As quatro respostas gravadas (critério 11, T29) entram quando a gravação chegar; se divergirem do dublê, o dublê muda. --- .../connectors/src/mercado-livre/fake-data.ts | 32 +++- .../src/mercado-livre/fake-routes-more.ts | 15 ++ .../src/mercado-livre/fake-routes.ts | 34 +++- .../src/mercado-livre/operations.test.ts | 177 +++++++++++++++++- .../src/mercado-livre/operations/anuncios.ts | 8 + .../mercado-livre/operations/faturamento.ts | 4 + .../src/mercado-livre/operations/perguntas.ts | 9 +- .../mercado-livre/operations/reclamacoes.ts | 4 + .../src/mercado-livre/operations/shared.ts | 15 +- .../src/mercado-livre/operations/vendas.ts | 4 + .../src/mercado-livre/skill/SKILL.md | 3 +- .../skill/references/anuncios.md | 2 +- .../skill/references/perguntas.md | 2 +- .../mercado-livre/skill/references/vendas.md | 2 +- 14 files changed, 298 insertions(+), 13 deletions(-) diff --git a/packages/connectors/src/mercado-livre/fake-data.ts b/packages/connectors/src/mercado-livre/fake-data.ts index d9992b67..99039e83 100644 --- a/packages/connectors/src/mercado-livre/fake-data.ts +++ b/packages/connectors/src/mercado-livre/fake-data.ts @@ -9,9 +9,10 @@ export const REDIRECT_URI = 'https://hub.exemplo.test/integrations/return' export const SELLER = Object.freeze({ ref: '123456789', label: 'PADARIASOL' }) +/** More than the 1,000 an offset search reaches: only a scan reads them all. */ export const ITEMS = Array.from( - { length: 12 }, - (_, index) => `MLB10000000${String(index + 1).padStart(2, '0')}`, + { length: 1300 }, + (_, index) => `MLB1000000${String(index + 1).padStart(4, '0')}`, ) export const ORDER = 2000000000001 export const PACK = 2000000000099 @@ -49,6 +50,33 @@ export const ORDER_BODY = { ], } +/** 120 orders, the first `ORDER_BODY`; each changed a minute after the one before, so a changed-since filter keeps the newest. */ +export const ORDERS = Array.from({ length: 120 }, (_, index) => ({ + ...ORDER_BODY, + id: ORDER + index, + date_last_updated: new Date(Date.UTC(2026, 8, 10, 13, index)).toISOString(), +})) + +/** The rows of a scan's page: its `scroll_id` carries where the next page starts (`-`). */ +export function scanOf(rows: readonly T[], query: URLSearchParams, route: string) { + const scroll = query.get('scroll_id') + const offset = scroll === null ? 0 : Number(scroll.slice(route.length + 1)) + const limit = Number(query.get('limit') ?? 50) + const page = rows.slice(offset, offset + limit) + return { rows: page, scrollId: `${route}-${offset + page.length}`, total: rows.length, limit } +} + +/** A scan's cursor the double never gave. */ +export function badScroll(query: URLSearchParams, route: string): boolean { + const scroll = query.get('scroll_id') + return scroll !== null && !new RegExp(`^${route}-\\d+$`).test(scroll) +} + +export const BAD_SCROLL = { + status: 400, + body: { message: 'invalid scroll_id', error: 'bad_request', status: 400 }, +} + export function itemBody(id: string, index: number) { return { id, diff --git a/packages/connectors/src/mercado-livre/fake-routes-more.ts b/packages/connectors/src/mercado-livre/fake-routes-more.ts index 56324cd0..65b30129 100644 --- a/packages/connectors/src/mercado-livre/fake-routes-more.ts +++ b/packages/connectors/src/mercado-livre/fake-routes-more.ts @@ -1,6 +1,8 @@ import type { Route } from './fake-data.ts' import { + BAD_SCROLL, BUYER, + badScroll, CLAIM, ITEMS, ORDER, @@ -9,6 +11,7 @@ import { paging, SELLER, SELLER_PATH, + scanOf, } from './fake-data.ts' const QUESTIONS = Array.from({ length: 12 }, (_, index) => ({ @@ -50,6 +53,18 @@ export const MORE_ROUTES: readonly Route[] = [ group: 'perguntas', pattern: /^\/questions\/search$/, answer: ({ query }) => { + if (query.get('search_type') === 'scan') { + if (badScroll(query, 'questions')) return BAD_SCROLL + const scan = scanOf(QUESTIONS, query, 'questions') + return { + body: { + total: scan.total, + limit: scan.limit, + questions: scan.rows, + scroll_id: scan.scrollId, + }, + } + } const page = pageOf(QUESTIONS, query, 50) return { body: { diff --git a/packages/connectors/src/mercado-livre/fake-routes.ts b/packages/connectors/src/mercado-livre/fake-routes.ts index 68b02c55..d3f9cf23 100644 --- a/packages/connectors/src/mercado-livre/fake-routes.ts +++ b/packages/connectors/src/mercado-livre/fake-routes.ts @@ -1,15 +1,19 @@ import type { Route } from './fake-data.ts' import { + BAD_SCROLL, BUYER, + badScroll, ITEMS, itemBody, ORDER, ORDER_BODY, + ORDERS, pageOf, paging, SELLER, SELLER_PATH, SHIPMENT, + scanOf, } from './fake-data.ts' import { MORE_ROUTES } from './fake-routes-more.ts' @@ -20,6 +24,18 @@ export const ROUTES: readonly Route[] = [ group: 'anuncios', pattern: new RegExp(`^/users/${SELLER_PATH}/items/search$`), answer: ({ query }) => { + if (query.get('search_type') === 'scan') { + if (badScroll(query, 'items')) return BAD_SCROLL + const scan = scanOf(ITEMS, query, 'items') + return { + body: { + seller_id: SELLER.ref, + results: scan.rows, + scroll_id: scan.scrollId, + paging: { total: scan.total, limit: scan.limit }, + }, + } + } const page = pageOf(ITEMS, query, 50) return { body: { @@ -84,9 +100,21 @@ export const ROUTES: readonly Route[] = [ name: 'pedidos', group: 'vendas', pattern: /^\/orders\/search$/, - answer: ({ query }) => ({ - body: { results: [ORDER_BODY], paging: paging(1, query), sort: { id: 'date_desc' } }, - }), + answer: ({ query }) => { + const from = query.get('order.date_last_updated.from') + const changed = + from === null + ? ORDERS + : ORDERS.filter((order) => Date.parse(order.date_last_updated) >= Date.parse(from)) + const page = pageOf(changed, query, 50) + return { + body: { + results: page.rows, + paging: paging(page.total, query), + sort: { id: 'date_desc' }, + }, + } + }, }, { name: 'pedido', diff --git a/packages/connectors/src/mercado-livre/operations.test.ts b/packages/connectors/src/mercado-livre/operations.test.ts index 3f3c39e9..657bc416 100644 --- a/packages/connectors/src/mercado-livre/operations.test.ts +++ b/packages/connectors/src/mercado-livre/operations.test.ts @@ -2,8 +2,13 @@ import assert from 'node:assert/strict' import { test } from 'node:test' import { SENT_AT } from '../kernel/conformance-declared.ts' import { answerOf } from '../kernel/conformance-session.ts' +import type { CopyDefinitionV2, CopyPosition } from '../kernel/copy.ts' +import { CopyDefinitionV2 as CopyDefinitionV2Schema } from '../kernel/copy.ts' +import { listInputOf, listPageOf } from '../kernel/list.ts' +import type { Operation } from '../kernel/operation.ts' import { requestOf } from '../kernel/operation.ts' import { toRequest } from '../kernel/request.ts' +import { Rows } from '../kernel/rows.ts' import { mercadoLivre } from './definition.ts' import { mercadoLivreFake } from './fake.ts' import { mensagensDoPacote } from './operations/mensagens.ts' @@ -210,7 +215,7 @@ async function twoPages( } for (const [id, total] of [ - ['anuncios', 12], + ['anuncios', 1300], ['perguntas', 12], ['periodosDeFaturamento', 14], ] as const) { @@ -315,3 +320,173 @@ test('o total e o temMais saem das mesmas contagens: sem deslocamento só o tota value: { linhas: [{ id: 'a' }], temMais: null }, }) }) + +/** Every page of a list read of the double, through the copy's own input and page: the cells of each row. */ +async function walked( + operation: Operation, + definition: CopyDefinitionV2, + since: string | null = null, +) { + const list = operation.list ?? assert.fail(`${operation.id} declares no list`) + const fake = mercadoLivreFake() + const granted = await authorize(fake) + if (granted.kind !== 'OK') return assert.fail('authorize') + const cells: (string | null)[][] = [] + let position: CopyPosition | null = null + for (let pages = 0; pages < 100; pages++) { + const input = operation.input.parse(listInputOf({ list, definition, position, since })) + const request = requestOf(operation, input, ACCOUNT, SENT_AT) + const answered = operation.answer( + await answerOf(await fake.handle(toRequest(mercadoLivre, request, granted.accessToken))), + ) + if (answered.kind !== 'OK') return assert.fail(`${operation.id}: ${answered.kind}`) + const rows = Rows.parse(answered.value) + const page = listPageOf({ list, definition, position, rows, at: SENT_AT }) + if (page.kind !== 'PAGE') return assert.fail(`${operation.id}: ${page.kind}`) + cells.push(...page.rows.map((row) => [...row])) + if (!page.more) return { cells, pages: pages + 1 } + position = page.next + } + return assert.fail(`${operation.id} never ends`) +} + +function definitionOf( + operation: Operation, + columns: { name: string; type: string; path: string }[], + changed: string | null = null, +) { + return CopyDefinitionV2Schema.parse({ + version: 2, + source: { kind: 'OPERATION', operation: operation.id, input: {} }, + description: 'Teste.', + columns, + key: [columns[0]?.name], + changedSince: changed === null ? null : { column: changed }, + }) +} + +test('pedidos declara a lista por deslocamento e o filtro do que mudou', async () => { + const pedidos = operationOf('pedidos') + assert.deepEqual(pedidos.list, { + paging: { kind: 'OFFSET', offset: 'offset', limit: 'limit', max: 50, ceiling: null }, + changed: { input: 'changedFrom', field: 'date_last_updated' }, + }) + const definition = definitionOf( + pedidos, + [ + { name: 'id', type: 'bigint', path: 'id' }, + { name: 'alterado', type: 'timestamp', path: 'date_last_updated' }, + ], + 'alterado', + ) + const all = await walked(pedidos, definition) + assert.equal(all.cells.length, 120, 'os 120 pedidos do dublê') + assert.equal(all.pages, 3) + const changed = await walked(pedidos, definition, '2026-09-10T14:50:00.000Z') + assert.deepEqual( + changed.cells.map((row) => row[1]), + all.cells + .map((row) => row[1]) + .filter((at) => Date.parse(at ?? '') >= Date.parse('2026-09-10T14:50:00.000Z')), + 'só o que mudou desde o instante', + ) + assert.equal(changed.cells.length, 10) +}) + +test('a varredura passa de mil', async () => { + for (const [id, max] of [ + ['anuncios', 100], + ['perguntas', 50], + ] as const) { + assert.deepEqual( + operationOf(id).list, + { + paging: { + kind: 'CURSOR', + cursor: 'scrollId', + limit: 'limit', + max, + start: { scan: true }, + ttlMs: 300_000, + }, + changed: null, + }, + id, + ) + } + const anuncios = operationOf('anuncios') + const all = await walked( + anuncios, + definitionOf(anuncios, [{ name: 'id', type: 'text', path: 'id' }]), + ) + assert.equal(all.cells.length, 1300, 'os 1.300 anúncios, passando de mil') + assert.equal(new Set(all.cells.map((row) => row[0])).size, 1300, 'nenhum repete') + const list = anuncios.list ?? assert.fail('list') + const definition = definitionOf(anuncios, [{ name: 'id', type: 'text', path: 'id' }]) + function pageOf(body: Record) { + const answered = scanned(['MLB1'], body) + if (answered.kind !== 'OK') return assert.fail(answered.kind) + return listPageOf({ + list, + definition, + position: null, + rows: Rows.parse(answered.value), + at: SENT_AT, + }) + } + assert.deepEqual( + pageOf({ paging: { total: 9, offset: 0, limit: 1 } }), + { kind: 'UNREADABLE', reason: 'NO_CURSOR' }, + 'linhas, mais por vir e sem scroll_id', + ) + assert.deepEqual( + pageOf({ paging: { total: 9, limit: 1 } }), + { kind: 'UNREADABLE', reason: 'NO_MORE_SIGNAL' }, + 'linhas sem scroll_id e sem deslocamento: nenhum sinal de fim', + ) +}) + +test('as outras listas por deslocamento, e as mensagens de um pacote sem lista', () => { + for (const [id, max] of [ + ['reclamacoes', 30], + ['periodosDeFaturamento', 12], + ] as const) + assert.deepEqual( + operationOf(id).list, + { + paging: { kind: 'OFFSET', offset: 'offset', limit: 'limit', max, ceiling: null }, + changed: null, + }, + id, + ) + assert.equal(mensagensDoPacote.list, undefined, 'pede o packId de um pacote') + const listed = mercadoLivre.operations.filter((operation) => operation.list !== undefined) + assert.deepEqual(listed.map((operation) => operation.id).sort(), [ + 'anuncios', + 'pedidos', + 'perguntas', + 'periodosDeFaturamento', + 'reclamacoes', + ]) +}) + +test('o comprador não chega à cópia', async () => { + const personal = ['maria.ficticia', '988776655', '52998224725', 'Fictícia', 'COMPRADORFICTICIO'] + for (const id of ['pedidos', 'perguntas']) { + const operation = operationOf(id) + const fields = ['id', 'buyer', 'from', 'status', 'order_items', 'text', 'date_created'] + const columns = [ + { name: 'id', type: 'bigint', path: 'id' }, + ...fields.slice(1).map((field) => ({ name: field, type: 'json', path: field })), + ] + const all = await walked(operation, definitionOf(operation, columns)) + assert.ok(all.cells.length > 0, id) + const text = JSON.stringify(all.cells) + for (const value of personal) assert.ok(!text.includes(value), `${id} copia ${value}`) + const at = (name: string) => columns.findIndex((column) => column.name === name) + assert.ok( + all.cells.every((row) => row[at(id === 'pedidos' ? 'buyer' : 'from')] === null), + `${id}: o objeto cortado é nulo`, + ) + } +}) diff --git a/packages/connectors/src/mercado-livre/operations/anuncios.ts b/packages/connectors/src/mercado-livre/operations/anuncios.ts index f108e4f6..96fc9849 100644 --- a/packages/connectors/src/mercado-livre/operations/anuncios.ts +++ b/packages/connectors/src/mercado-livre/operations/anuncios.ts @@ -7,6 +7,7 @@ import { Offset, PathId, readOf, + SCROLL_TTL_MS, Scan, searchTypeOf, sellerOf, @@ -29,6 +30,13 @@ export const anuncios = readOf({ limit: Limit(100), }), list: 'results', + paged: { + paging: { + ...{ kind: 'CURSOR', cursor: 'scrollId', limit: 'limit', max: 100 }, + ...{ start: { scan: true }, ttlMs: SCROLL_TTL_MS }, + }, + changed: null, + }, probe: () => ({ offset: 0, limit: 5 }), request: (input, account) => get(`/users/${sellerOf(account)}/items/search`, { diff --git a/packages/connectors/src/mercado-livre/operations/faturamento.ts b/packages/connectors/src/mercado-livre/operations/faturamento.ts index 05a5cee4..b013e9ea 100644 --- a/packages/connectors/src/mercado-livre/operations/faturamento.ts +++ b/packages/connectors/src/mercado-livre/operations/faturamento.ts @@ -19,6 +19,10 @@ export const periodosDeFaturamento = readOf({ limit: Limit(12).optional(), }), list: 'results', + paged: { + paging: { kind: 'OFFSET', offset: 'offset', limit: 'limit', max: 12, ceiling: null }, + changed: null, + }, probe: () => ({}), request: (input) => get('/billing/integration/monthly/periods', { diff --git a/packages/connectors/src/mercado-livre/operations/perguntas.ts b/packages/connectors/src/mercado-livre/operations/perguntas.ts index 570d5469..fb49fa2f 100644 --- a/packages/connectors/src/mercado-livre/operations/perguntas.ts +++ b/packages/connectors/src/mercado-livre/operations/perguntas.ts @@ -1,6 +1,6 @@ import { z } from 'zod' import { get } from '../../kernel/operation.ts' -import { Limit, Offset, readOf, Scan, searchTypeOf, sellerOf } from './shared.ts' +import { Limit, Offset, readOf, SCROLL_TTL_MS, Scan, searchTypeOf, sellerOf } from './shared.ts' export const perguntas = readOf({ id: 'perguntas', @@ -26,6 +26,13 @@ export const perguntas = readOf({ limit: Limit(50), }), list: 'questions', + paged: { + paging: { + ...{ kind: 'CURSOR', cursor: 'scrollId', limit: 'limit', max: 50 }, + ...{ start: { scan: true }, ttlMs: SCROLL_TTL_MS }, + }, + changed: null, + }, omit: ['from'], // the asker: the question route adds their name and contact probe: () => ({ offset: 0, limit: 5 }), request: (input, account) => diff --git a/packages/connectors/src/mercado-livre/operations/reclamacoes.ts b/packages/connectors/src/mercado-livre/operations/reclamacoes.ts index 7cb1db4d..32fa4eb3 100644 --- a/packages/connectors/src/mercado-livre/operations/reclamacoes.ts +++ b/packages/connectors/src/mercado-livre/operations/reclamacoes.ts @@ -15,6 +15,10 @@ export const reclamacoes = readOf({ limit: Limit(30), }), list: 'data', + paged: { + paging: { kind: 'OFFSET', offset: 'offset', limit: 'limit', max: 30, ceiling: null }, + changed: null, + }, probe: () => ({ status: 'opened' as const, offset: 0, limit: 5 }), request: (input, account) => get('/post-purchase/v1/claims/search', { diff --git a/packages/connectors/src/mercado-livre/operations/shared.ts b/packages/connectors/src/mercado-livre/operations/shared.ts index c4c54bdd..37c43e88 100644 --- a/packages/connectors/src/mercado-livre/operations/shared.ts +++ b/packages/connectors/src/mercado-livre/operations/shared.ts @@ -1,5 +1,11 @@ import { z } from 'zod' -import type { Account, Operation, OperationId, ProviderRequest } from '../../kernel/operation.ts' +import type { + Account, + ListRead, + Operation, + OperationId, + ProviderRequest, +} from '../../kernel/operation.ts' import { defineOperation, valueAt } from '../../kernel/operation.ts' import type { Rows } from '../../kernel/rows.ts' import { Rows as RowsOut } from '../../kernel/rows.ts' @@ -19,6 +25,9 @@ export const Scan = { scrollId: z.string().min(1).max(4000).optional(), } +/** A scan's `scroll_id` lives 5 minutes (itens-e-buscas): the copy never sends an older one. */ +export const SCROLL_TTL_MS = 300_000 + /** The `search_type` the route wants while a scan runs: from its first page on. */ export function searchTypeOf(input: Readonly<{ scan?: true; scrollId?: string }>) { return input.scan === true || input.scrollId !== undefined ? 'scan' : undefined @@ -44,6 +53,7 @@ type ReadSpec = Readonly<{ label: string input: z.ZodType list: string | null // where the rows are: a key of the body, or the body itself + paged?: ListRead // how the list pages and asks only what changed: the copy engine reads it as the operation's `list` omit?: readonly string[] // the buyer's personal data, by dotted path emptyNotFound?: string // the start of the message of the 404 that means the seller has no data yet from?: readonly string[] @@ -53,9 +63,10 @@ type ReadSpec = Readonly<{ /** A read of Mercado Livre: idempotent by class, answered as rows by the API's own table (`rows`). */ export function readOf(spec: ReadSpec): Operation { - const { docs, list, omit, emptyNotFound, ...operation } = spec + const { docs, list, omit, emptyNotFound, paged, ...operation } = spec const defined = defineOperation({ ...operation, + ...(paged === undefined ? {} : { list: paged }), class: 'read' as const, idempotent: true, ...rows({ list, omit, emptyNotFound }), diff --git a/packages/connectors/src/mercado-livre/operations/vendas.ts b/packages/connectors/src/mercado-livre/operations/vendas.ts index 5e26f51b..782813ce 100644 --- a/packages/connectors/src/mercado-livre/operations/vendas.ts +++ b/packages/connectors/src/mercado-livre/operations/vendas.ts @@ -26,6 +26,10 @@ export const pedidos = readOf({ limit: Limit(50), }), list: 'results', + paged: { + paging: { kind: 'OFFSET', offset: 'offset', limit: 'limit', max: 50, ceiling: null }, + changed: { input: 'changedFrom', field: 'date_last_updated' }, + }, omit: BUYER, probe: () => ({ offset: 0, limit: 5 }), request: (input, account) => diff --git a/packages/connectors/src/mercado-livre/skill/SKILL.md b/packages/connectors/src/mercado-livre/skill/SKILL.md index 1e424dc8..426ae6c4 100644 --- a/packages/connectors/src/mercado-livre/skill/SKILL.md +++ b/packages/connectors/src/mercado-livre/skill/SKILL.md @@ -12,7 +12,8 @@ The connection is read only. There is no operation that writes, and a refusal is ## Rules for every read - Every read answers `{ linhas, temMais }`: the rows as Mercado Livre sent them (a field the kit does not know passes, and one object comes as a list of one), and whether another page follows (`temMais` is `null` when the route does not say). The app parses the fields it uses at its own boundary, never the kit. -- One page per call. A list ends when `offset + limit` reaches `total` (`paging`, or the same counts at the top of the answer), and `temMais` says so. Read the next page only when the person's question needs it. +- One page per call. A list ends when `offset + limit` reaches `total` (`paging`, or the same counts at the top of the answer), and `temMais` says so; a scan (`anuncios`, `perguntas`) gives the next page's cursor in `seguinte` and ends on a page with no rows. Read the next page only when the person's question needs it. +- `pedidos`, `anuncios`, `perguntas`, `reclamacoes` and `periodosDeFaturamento` declare how their list pages (and `pedidos`, how it asks only what changed), so each can become a copy the Conexus keeps in the company's database: for data an app reads again and again, propose the copy instead of paging live. - Each list has its own page size cap and its own window; the part for the subject gives them. - A refusal comes with what Mercado Livre answered. A permission the app lacks (`PA_UNAUTHORIZED_RESULT_FROM_POLICIES`) refuses only that group of reads; the others still work. - An answer Mercado Livre could not fill whole (a partial `206`, on an order) is read, and carries `parcial: true`: some fields are missing, and Mercado Livre says the rest usually suffices. Say so when a missing field matters to the question. diff --git a/packages/connectors/src/mercado-livre/skill/references/anuncios.md b/packages/connectors/src/mercado-livre/skill/references/anuncios.md index 0784b0f1..b6405d59 100644 --- a/packages/connectors/src/mercado-livre/skill/references/anuncios.md +++ b/packages/connectors/src/mercado-livre/skill/references/anuncios.md @@ -1,6 +1,6 @@ # Anúncios -- `anuncios`: the ids of the seller's listings, filtered by `status` or `sku`. Up to 100 per page (`offset`, `limit`). The search reads at most 1000 listings (Mercado Livre's offset cap): a seller with more narrows it with `status` or `sku`; there is no way past 1000. +- `anuncios`: the ids of the seller's listings, filtered by `status` or `sku`. Up to 100 per page. By `offset` the search reads at most 1000 listings (Mercado Livre's offset cap); a scan reads them all: the first page sends `scan: true`, each answer gives the next page's cursor in `seguinte`, sent back as `scrollId` (it lives 5 minutes), and a page with no rows is the end. - `anunciosDetalhe`: up to 20 listings per call by id (title, price, stock, sold, status, category, listing type). Each item has its own `code`: a 404 there is that id's, not the call's. - `descricao`: the plain-text description of one listing. - `estoque`: the stock of one user product by warehouse. Limit of 100 calls per minute. diff --git a/packages/connectors/src/mercado-livre/skill/references/perguntas.md b/packages/connectors/src/mercado-livre/skill/references/perguntas.md index 8c056edb..d402e195 100644 --- a/packages/connectors/src/mercado-livre/skill/references/perguntas.md +++ b/packages/connectors/src/mercado-livre/skill/references/perguntas.md @@ -1,4 +1,4 @@ # Perguntas -- `perguntas`: questions on the seller's listings with the answer when there is one, up to 50 per page, filtered by `status`. Unanswered questions stay visible for 7 months. The asker is not in the answer. The search reads at most 1000 questions (Mercado Livre's offset cap): with more, narrow it with `status`; there is no way past 1000. +- `perguntas`: questions on the seller's listings with the answer when there is one, up to 50 per page, filtered by `status`. Unanswered questions stay visible for 7 months. The asker is not in the answer. By `offset` the search reads at most 1000 questions (Mercado Livre's offset cap); a scan reads them all: the first page sends `scan: true`, each answer gives the next page's cursor in `seguinte`, sent back as `scrollId` (it lives 5 minutes), and a page with no rows is the end. - `tempoDeResposta`: the seller's average answer time, in minutes. diff --git a/packages/connectors/src/mercado-livre/skill/references/vendas.md b/packages/connectors/src/mercado-livre/skill/references/vendas.md index 03c834ed..64716743 100644 --- a/packages/connectors/src/mercado-livre/skill/references/vendas.md +++ b/packages/connectors/src/mercado-livre/skill/references/vendas.md @@ -1,6 +1,6 @@ # Vendas -- `pedidos`: the seller's orders, up to 50 per page. Orders are kept for 12 months; filter by `status` and by `dateFrom`/`dateTo` (ISO 8601 with offset) and sort with `date_asc` or `date_desc`. An order can come partial (`206`): it is read, and `parcial: true` says some of its fields are missing. +- `pedidos`: the seller's orders, up to 50 per page. Orders are kept for 12 months; filter by `status` and by `dateFrom`/`dateTo` (ISO 8601 with offset) and sort with `date_asc` or `date_desc`. `changedFrom` (ISO 8601 with offset) asks only the orders changed since that instant (`date_last_updated`). An order can come partial (`206`): it is read, and `parcial: true` says some of its fields are missing. - `pedido`: one order by id, with its items, its `pack_id` and its shipping id. - `pacote`: one pack (several orders of one buyer shipped together). Only orders with a `pack_id` have one. From ac1d51544be48e7a8e9f4032bba703022fceab4d Mon Sep 17 00:00:00 2001 From: developmentconexus Date: Wed, 11 Nov 2026 21:21:59 -0300 Subject: [PATCH 4/8] feat(connectors): o gravador de conectores imprime o plano dos quatro pedidos e a troca aceita o corpo gravado longe, com a marca no lugar do vendedor --- scripts/record-connector-calls.ts | 57 +++++++++++++++++++++++++++---- scripts/record-connector-swap.ts | 6 +++- scripts/record-connector.test.ts | 30 +++++++++++++++- scripts/record-connector.ts | 31 +++++++++++------ 4 files changed, 106 insertions(+), 18 deletions(-) diff --git a/scripts/record-connector-calls.ts b/scripts/record-connector-calls.ts index 0702b655..e1feac61 100644 --- a/scripts/record-connector-calls.ts +++ b/scripts/record-connector-calls.ts @@ -3,7 +3,7 @@ // the Hub's own `bodyOf`, kept before `operation.answer` reads them, as Nango's interceptor keeps // `response.data` (dryrun.service.ts:468-475). The credential is a parameter and never leaves it. import { isAbsolute, relative, resolve } from 'node:path' -import type { Connector } from '@conexus/connectors' +import type { Connector, ProviderRequest } from '@conexus/connectors' import { AccountRef, requestOf, toRequest } from '@conexus/connectors' import { z } from 'zod' import { bodyOf } from '../apps/server/src/hub/connectors/wire.ts' @@ -31,6 +31,7 @@ export const Raw = z.object({ export type Raw = z.infer const PAGE_1 = 'anuncios-varredura-1' +const WEEK_MS = 7 * 24 * 60 * 60 * 1000 /** The operation of a case and its input; the second listing page carries the first page's `scroll_id`. */ export function inputOf( @@ -39,7 +40,7 @@ export function inputOf( first: unknown, ): { operation: string; input: unknown } { if (name === 'pedidos-alterados') { - const week = new Date(now.getTime() - 7 * 24 * 60 * 60 * 1000) + const week = new Date(now.getTime() - WEEK_MS) return { operation: 'pedidos', input: { changedFrom: week.toISOString(), offset: 0, limit: 50 }, @@ -63,6 +64,53 @@ export function assertOutside(out: string, root: string): string { return path } +/** The request a case sends, as the operation declares it. */ +function requestFor( + connector: Connector, + name: Case, + account: string, + now: Date, + first: unknown, +): ProviderRequest { + const { operation: id, input } = inputOf(name, now, first) + const operation = connector.operations.find((candidate) => candidate.id === id) + if (operation === undefined) throw new Error(`${connector.id} declares no ${id}`) + return requestOf(operation, input, { ref: AccountRef.parse(account) }, now) +} + +/** The seller's id in a raw file that was recorded away from this machine: the file never carries it. */ +export const ACCOUNT = 'ACCOUNT' +const PLAN_NOW = new Date('2026-01-15T12:00:00.000Z') + +/** + * The four requests as the operations declare them, for a recorder that runs where Node does not: + * the seller's id, the changed-from date and the first page's `scroll_id` stay as marks the other + * side fills. The credential is not part of it. + */ +export function planned(connector: Connector) { + const changed = new Date(PLAN_NOW.getTime() - WEEK_MS).toISOString() + return { + changedFromDaysAgo: WEEK_MS / (24 * 60 * 60 * 1000), + cases: CASES.map((name) => { + const request = requestFor(connector, name, ACCOUNT, PLAN_NOW, { scroll_id: '{scroll_id}' }) + const query = Object.fromEntries( + Object.entries(request.query).map(([key, value]) => [ + key, + value === changed ? '{changed_from}' : value, + ]), + ) + return { + case: name, + method: request.method, + path: request.path, + query, + headers: request.headers ?? {}, + origin: connector.origin, + } + }), + } +} + /** * Sends one case and returns what arrived. A refusal or an unreadable body is thrown by status * only, nothing is returned, and the credential and the account are never in the message. @@ -78,10 +126,7 @@ export async function recorded( first: unknown }>, ): Promise { - const { operation: id, input } = inputOf(a.name, a.now, a.first) - const operation = a.connector.operations.find((candidate) => candidate.id === id) - if (operation === undefined) throw new Error(`${a.connector.id} declares no ${id}`) - const request = requestOf(operation, input, { ref: AccountRef.parse(a.account) }, a.now) + const request = requestFor(a.connector, a.name, a.account, a.now, a.first) const response = await a.fetch(toRequest(a.connector, request, a.token), { signal: AbortSignal.timeout(60_000), }) diff --git a/scripts/record-connector-swap.ts b/scripts/record-connector-swap.ts index dc36434b..bdc50d68 100644 --- a/scripts/record-connector-swap.ts +++ b/scripts/record-connector-swap.ts @@ -118,7 +118,11 @@ export function swapped(raw: Readonly<{ body: unknown } & Seed>): Swapped { const latest = Math.max(...instantsOf(raw.body), ANCHOR) const by = ANCHOR - latest const body = walk(raw.body, '', [], { invented, by }) - const account = String(invented.of(Number(raw.account), 'seller', ['seller'])) + const account = String( + invented.of(/^\d+$/.test(raw.account) ? Number(raw.account) : raw.account, 'seller', [ + 'seller', + ]), + ) const path = raw.path.replaceAll(raw.account, account) const left = survivors(raw.body, body) if (left.length > 0) diff --git a/scripts/record-connector.test.ts b/scripts/record-connector.test.ts index ea5b8092..a2d408bd 100644 --- a/scripts/record-connector.test.ts +++ b/scripts/record-connector.test.ts @@ -2,7 +2,7 @@ import assert from 'node:assert/strict' import { test } from 'node:test' import { CONNECTORS } from '@conexus/connectors' import { z } from 'zod' -import { assertOutside, recorded } from './record-connector-calls.ts' +import { ACCOUNT, assertOutside, planned, recorded } from './record-connector-calls.ts' import { swapped } from './record-connector-swap.ts' const ML = CONNECTORS.find((connector) => connector.id === 'mercado-livre') @@ -157,3 +157,31 @@ test('um endereço ou prefixo de credencial que sobra recusa a gravação', () = /nothing is saved/, ) }) + +test('o plano traz os quatro pedidos da declaração com marcas no lugar do vendedor, da data e do scroll_id', () => { + const plan = planned(ML) + assert.deepEqual( + plan.cases.map((one) => `${one.method} ${one.path}`), + [ + `GET /orders/search`, + `GET /users/${ACCOUNT}/items/search`, + `GET /users/${ACCOUNT}/items/search`, + `GET /questions/search`, + ], + ) + assert.equal(plan.changedFromDaysAgo, 7) + assert.equal(plan.cases[0]?.query['order.date_last_updated.from'], '{changed_from}') + assert.equal(plan.cases[0]?.query.seller, ACCOUNT) + assert.equal(plan.cases[2]?.query.scroll_id, '{scroll_id}') + assert.equal(plan.cases[3]?.query.search_type, 'scan') +}) + +test('um arquivo cru gravado longe, com a marca no lugar do vendedor, troca o caminho sem o id real', () => { + const done = swapped({ + body: { results: [{ id: 2000007654321, seller: { id: 987654321 } }] }, + account: ACCOUNT, + path: `/users/${ACCOUNT}/items/search`, + }) + assert.equal(done.path.includes(`${ACCOUNT}/`), false) + assert.match(done.path, /^\/users\/[A-Za-z0-9]+\/items\/search$/) +}) diff --git a/scripts/record-connector.ts b/scripts/record-connector.ts index d0f8b1bc..434ede37 100644 --- a/scripts/record-connector.ts +++ b/scripts/record-connector.ts @@ -8,6 +8,9 @@ // `... --case --out --swap` replaces every value of that raw body by an invented // one of the same shape and saves the recording in `packages/connectors/src//recordings/`, // with the service, the route, the date and the paths it replaced beside it. +// `... --plan` prints the four requests as the operations declare them, with the seller's id, the +// changed-from date and the first page's `scroll_id` left as marks, for `gravar_ml.py`, the same four +// reads from a machine where only Python runs; the raw files it keeps carry `ACCOUNT` where the id was. // The credential comes from the environment only and never reaches a file or the output (security // S40): CONEXUS_RECORD_ML_TOKEN (a seller's access token) and CONEXUS_RECORD_ML_SELLER (the // seller's id, the account the read is for). @@ -16,18 +19,20 @@ import { join } from 'node:path' import { parseArgs } from 'node:util' import { CONNECTORS } from '@conexus/connectors' import { z } from 'zod' -import { assertOutside, CASES, Raw, recorded } from './record-connector-calls.ts' +import { ACCOUNT, assertOutside, CASES, planned, Raw, recorded } from './record-connector-calls.ts' import { swapped } from './record-connector-swap.ts' const ROOT = join(import.meta.dirname, '..') const Args = z.object({ connector: z.literal('mercado-livre'), - case: z.enum(CASES), - out: z.string().min(1), + case: z.enum(CASES).optional(), + out: z.string().min(1).optional(), swap: z.boolean().default(false), + plan: z.boolean().default(false), }) const Seller = z.object({ CONEXUS_RECORD_ML_SELLER: z.string().min(1) }) const Env = Seller.extend({ CONEXUS_RECORD_ML_TOKEN: z.string().min(1) }) +const Run = z.object({ case: z.enum(CASES), out: z.string().min(1) }) const { values } = parseArgs({ options: { @@ -35,19 +40,25 @@ const { values } = parseArgs({ case: { type: 'string' }, out: { type: 'string' }, swap: { type: 'boolean' }, + plan: { type: 'boolean' }, }, }) const args = Args.parse(values) -const out = assertOutside(args.out, ROOT) const connector = CONNECTORS.find((candidate) => candidate.id === args.connector) if (connector === undefined) throw new Error(`No connector ${args.connector}`) +if (args.plan) { + console.log(JSON.stringify(planned(connector), null, 2)) + process.exit(0) +} +const run = Run.parse(args) +const out = assertOutside(run.out, ROOT) function rawPath(name: string) { return join(out, `${name}.raw.json`) } if (args.swap) { - const raw = Raw.parse(JSON.parse(readFileSync(rawPath(args.case), 'utf8'))) - const { CONEXUS_RECORD_ML_SELLER: account } = Seller.parse(process.env) + const raw = Raw.parse(JSON.parse(readFileSync(rawPath(run.case), 'utf8'))) + const account = Seller.safeParse(process.env).data?.CONEXUS_RECORD_ML_SELLER ?? ACCOUNT const done = swapped({ body: raw.body, account, path: raw.path }) const folder = join(ROOT, 'packages/connectors/src', connector.id, 'recordings') mkdirSync(folder, { recursive: true }) @@ -66,19 +77,19 @@ if (args.swap) { } else { const env = Env.parse(process.env) const first = - args.case === 'anuncios-varredura-2' + run.case === 'anuncios-varredura-2' ? Raw.parse(JSON.parse(readFileSync(rawPath('anuncios-varredura-1'), 'utf8'))).body : null const raw = await recorded({ fetch, connector, - name: args.case, + name: run.case, account: env.CONEXUS_RECORD_ML_SELLER, token: env.CONEXUS_RECORD_ML_TOKEN, now: new Date(), first, }) mkdirSync(out, { recursive: true }) - writeFileSync(rawPath(args.case), JSON.stringify(raw, null, 2)) - console.log(`Kept ${args.case}.raw.json (status ${raw.status}) in the folder.`) + writeFileSync(rawPath(run.case), JSON.stringify(raw, null, 2)) + console.log(`Kept ${run.case}.raw.json (status ${raw.status}) in the folder.`) } From d0677e9274056915eece94eb3da178b9004a87db Mon Sep 17 00:00:00 2001 From: developmentconexus Date: Wed, 11 Nov 2026 22:05:31 -0300 Subject: [PATCH 5/8] feat(connectors): as quatro respostas gravadas do Mercado Livre entram com os valores trocados, e a troca cobre o id da conta que chama --- .../recordings/anuncios-varredura-1.json | 205 ++++++++ .../recordings/anuncios-varredura-2.json | 115 +++++ .../recordings/pedidos-alterados.json | 438 ++++++++++++++++++ .../recordings/perguntas-varredura.json | 117 +++++ scripts/record-connector-swap.ts | 4 +- scripts/record-connector.test.ts | 15 + 6 files changed, 892 insertions(+), 2 deletions(-) create mode 100644 packages/connectors/src/mercado-livre/recordings/anuncios-varredura-1.json create mode 100644 packages/connectors/src/mercado-livre/recordings/anuncios-varredura-2.json create mode 100644 packages/connectors/src/mercado-livre/recordings/pedidos-alterados.json create mode 100644 packages/connectors/src/mercado-livre/recordings/perguntas-varredura.json diff --git a/packages/connectors/src/mercado-livre/recordings/anuncios-varredura-1.json b/packages/connectors/src/mercado-livre/recordings/anuncios-varredura-1.json new file mode 100644 index 00000000..7806ad87 --- /dev/null +++ b/packages/connectors/src/mercado-livre/recordings/anuncios-varredura-1.json @@ -0,0 +1,205 @@ +{ + "service": "Mercado Livre", + "connector": "mercado-livre", + "case": "anuncios-varredura-1", + "route": { + "method": "GET", + "path": "/users/ACCOUNT/items/search", + "query": [ + "limit", + "search_type" + ] + }, + "recordedAt": "2026-11-12", + "status": 200, + "replaced": [ + "available_orders.id", + "available_orders.id.id", + "available_orders.id.missing", + "available_orders.name", + "orders.id", + "orders.name", + "results", + "scroll_id", + "seller", + "seller_id" + ], + "body": { + "scroll_id": "EYJPZCI0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001", + "seller_id": "000000002", + "results": [ + "Texto inventado 3", + "Texto inventado 4", + "Texto inventado 5", + "Texto inventado 6", + "Texto inventado 7", + "Texto inventado 8", + "Texto inventado 9", + "Texto inventado 10", + "Texto inventado 11", + "Texto inventado 12", + "Texto inventado 13", + "Texto inventado 14", + "Texto inventado 15", + "Texto inventado 16", + "Texto inventado 17", + "Texto inventado 18", + "Texto inventado 19", + "Texto inventado 20", + "Texto inventado 21", + "Texto inventado 22", + "Texto inventado 23", + "Texto inventado 24", + "Texto inventado 25", + "Texto inventado 26", + "Texto inventado 27", + "Texto inventado 28", + "Texto inventado 29", + "Texto inventado 30", + "Texto inventado 31", + "Texto inventado 32", + "Texto inventado 33", + "Texto inventado 34", + "Texto inventado 35", + "Texto inventado 36", + "Texto inventado 37", + "Texto inventado 38", + "Texto inventado 39", + "Texto inventado 40", + "Texto inventado 41", + "Texto inventado 42", + "Texto inventado 43", + "Texto inventado 44", + "Texto inventado 45", + "Texto inventado 46", + "Texto inventado 47", + "Texto inventado 48", + "Texto inventado 49", + "Texto inventado 50", + "Texto inventado 51", + "Texto inventado 52", + "Texto inventado 53", + "Texto inventado 54", + "Texto inventado 55", + "Texto inventado 56", + "Texto inventado 57", + "Texto inventado 58", + "Texto inventado 59", + "Texto inventado 60", + "Texto inventado 61", + "Texto inventado 62", + "Texto inventado 63", + "Texto inventado 64", + "Texto inventado 65", + "Texto inventado 66", + "Texto inventado 67", + "Texto inventado 68", + "Texto inventado 69", + "Texto inventado 70", + "Texto inventado 71", + "Texto inventado 72", + "Texto inventado 73", + "Texto inventado 74", + "Texto inventado 75", + "Texto inventado 76", + "Texto inventado 77", + "Texto inventado 78", + "Texto inventado 79", + "Texto inventado 80", + "Texto inventado 81", + "Texto inventado 82", + "Texto inventado 83", + "Texto inventado 84", + "Texto inventado 85", + "Texto inventado 86", + "Texto inventado 87", + "Texto inventado 88", + "Texto inventado 89", + "Texto inventado 90", + "Texto inventado 91" + ], + "paging": { + "limit": 100, + "total": 89 + }, + "query": null, + "orders": [ + { + "id": "STOPTIMEASC000092", + "name": "Texto inventado 93" + } + ], + "available_orders": [ + { + "id": "STOPTIMEASC000092", + "name": "Texto inventado 93" + }, + { + "id": "STOPTIMEDESC000094", + "name": "Texto inventado 95" + }, + { + "id": "STARTTIMEASC000096", + "name": "Texto inventado 97" + }, + { + "id": "STARTTIMEDESC000098", + "name": "Texto inventado 99" + }, + { + "id": "AVAILABLEQUANTITYASC000100", + "name": "Texto inventado 101" + }, + { + "id": "AVAILABLEQUANTITYDESC000102", + "name": "Texto inventado 103" + }, + { + "id": "SOLDQUANTITYASC000104", + "name": "Texto inventado 105" + }, + { + "id": "SOLDQUANTITYDESC000106", + "name": "Texto inventado 107" + }, + { + "id": "PRICEASC000108", + "name": "Texto inventado 109" + }, + { + "id": "PRICEDESC000110", + "name": "Texto inventado 111" + }, + { + "id": "LASTUPDATEDDESC000112", + "name": "Texto inventado 113" + }, + { + "id": "LASTUPDATEDASC000114", + "name": "Texto inventado 115" + }, + { + "id": "TOTALSOLDQUANTITYASC000116", + "name": "Texto inventado 117" + }, + { + "id": { + "id": "TOTALSOLDQUANTITYDESC000118", + "field": "sold_quantity", + "missing": "Texto inventado 119", + "order": "desc" + }, + "name": "Texto inventado 120" + }, + { + "id": { + "id": "INVENTORYIDASC000121", + "field": "inventory_id", + "missing": "Texto inventado 119", + "order": "asc" + }, + "name": "Texto inventado 122" + } + ] + } +} diff --git a/packages/connectors/src/mercado-livre/recordings/anuncios-varredura-2.json b/packages/connectors/src/mercado-livre/recordings/anuncios-varredura-2.json new file mode 100644 index 00000000..051f486a --- /dev/null +++ b/packages/connectors/src/mercado-livre/recordings/anuncios-varredura-2.json @@ -0,0 +1,115 @@ +{ + "service": "Mercado Livre", + "connector": "mercado-livre", + "case": "anuncios-varredura-2", + "route": { + "method": "GET", + "path": "/users/ACCOUNT/items/search", + "query": [ + "limit", + "scroll_id", + "search_type" + ] + }, + "recordedAt": "2026-11-12", + "status": 200, + "replaced": [ + "available_orders.id", + "available_orders.id.id", + "available_orders.id.missing", + "available_orders.name", + "orders.id", + "orders.name", + "scroll_id", + "seller", + "seller_id" + ], + "body": { + "scroll_id": "000001", + "seller_id": "000000002", + "results": [], + "paging": { + "limit": 100, + "total": 89 + }, + "query": null, + "orders": [ + { + "id": "STOPTIMEASC000003", + "name": "Texto inventado 4" + } + ], + "available_orders": [ + { + "id": "STOPTIMEASC000003", + "name": "Texto inventado 4" + }, + { + "id": "STOPTIMEDESC000005", + "name": "Texto inventado 6" + }, + { + "id": "STARTTIMEASC000007", + "name": "Texto inventado 8" + }, + { + "id": "STARTTIMEDESC000009", + "name": "Texto inventado 10" + }, + { + "id": "AVAILABLEQUANTITYASC000011", + "name": "Texto inventado 12" + }, + { + "id": "AVAILABLEQUANTITYDESC000013", + "name": "Texto inventado 14" + }, + { + "id": "SOLDQUANTITYASC000015", + "name": "Texto inventado 16" + }, + { + "id": "SOLDQUANTITYDESC000017", + "name": "Texto inventado 18" + }, + { + "id": "PRICEASC000019", + "name": "Texto inventado 20" + }, + { + "id": "PRICEDESC000021", + "name": "Texto inventado 22" + }, + { + "id": "LASTUPDATEDDESC000023", + "name": "Texto inventado 24" + }, + { + "id": "LASTUPDATEDASC000025", + "name": "Texto inventado 26" + }, + { + "id": "TOTALSOLDQUANTITYASC000027", + "name": "Texto inventado 28" + }, + { + "id": { + "id": "TOTALSOLDQUANTITYDESC000029", + "field": "sold_quantity", + "missing": "Texto inventado 30", + "order": "desc" + }, + "name": "Texto inventado 31" + }, + { + "id": { + "id": "INVENTORYIDASC000032", + "field": "inventory_id", + "missing": "Texto inventado 30", + "order": "asc" + }, + "name": "Texto inventado 33" + } + ] + } +} diff --git a/packages/connectors/src/mercado-livre/recordings/pedidos-alterados.json b/packages/connectors/src/mercado-livre/recordings/pedidos-alterados.json new file mode 100644 index 00000000..8585cea4 --- /dev/null +++ b/packages/connectors/src/mercado-livre/recordings/pedidos-alterados.json @@ -0,0 +1,438 @@ +{ + "service": "Mercado Livre", + "connector": "mercado-livre", + "case": "pedidos-alterados", + "route": { + "method": "GET", + "path": "/orders/search", + "query": [ + "limit", + "offset", + "order.date_last_updated.from", + "seller" + ] + }, + "recordedAt": "2026-11-12", + "status": 200, + "replaced": [ + "available_sorts.id", + "available_sorts.name", + "query", + "results.buyer.id", + "results.buyer.nickname", + "results.cancel_detail.application_id", + "results.cancel_detail.date", + "results.cancel_detail.description", + "results.context.site", + "results.coupon.amount", + "results.currency_id", + "results.date_closed", + "results.date_created", + "results.date_last_updated", + "results.expiration_date", + "results.id", + "results.last_updated", + "results.mediations.id", + "results.order_items.currency_id", + "results.order_items.element_id", + "results.order_items.gross_price", + "results.order_items.item.category_id", + "results.order_items.item.id", + "results.order_items.item.seller_custom_field", + "results.order_items.item.seller_sku", + "results.order_items.item.title", + "results.order_items.item.variation_attributes.id", + "results.order_items.item.variation_attributes.name", + "results.order_items.item.variation_attributes.value_id", + "results.order_items.item.variation_attributes.value_name", + "results.order_items.item.warranty", + "results.order_items.listing_type_id", + "results.order_items.sale_fee", + "results.order_items.unit_price", + "results.pack_id", + "results.paid_amount", + "results.payments.authorization_code", + "results.payments.available_actions", + "results.payments.card_id", + "results.payments.collector.id", + "results.payments.coupon_amount", + "results.payments.currency_id", + "results.payments.date_approved", + "results.payments.date_created", + "results.payments.date_last_modified", + "results.payments.id", + "results.payments.installment_amount", + "results.payments.issuer_id", + "results.payments.order_id", + "results.payments.overpaid_amount", + "results.payments.payer_id", + "results.payments.payment_method_id", + "results.payments.reason", + "results.payments.shipping_cost", + "results.payments.site_id", + "results.payments.taxes_amount", + "results.payments.total_paid_amount", + "results.payments.transaction_amount", + "results.payments.transaction_amount_refunded", + "results.seller.id", + "results.seller.nickname", + "results.shipping.id", + "results.total_amount", + "seller", + "sort.id", + "sort.name" + ], + "body": { + "query": "Texto inventado 1", + "results": [ + { + "payments": [ + { + "reason": "Texto inventado 2", + "status_code": null, + "total_paid_amount": 32.05, + "operation_type": "regular_payment", + "transaction_amount": 32.05, + "transaction_amount_refunded": 6, + "date_approved": "2026-08-04T06:40:50.000-04:00", + "collector": { + "id": 200000005 + }, + "coupon_id": null, + "installments": 1, + "authorization_code": null, + "taxes_amount": 6, + "id": 200000000006, + "date_last_modified": "2026-08-04T06:40:50.000-04:00", + "coupon_amount": 6, + "available_actions": [ + "refund" + ], + "shipping_cost": 6, + "installment_amount": null, + "date_created": "2026-08-04T06:40:50.000-04:00", + "activation_uri": null, + "overpaid_amount": 6, + "card_id": null, + "status_detail": "accredited", + "issuer_id": "000007", + "payment_method_id": "ACCOUNTMONEY000008", + "payment_type": "account_money", + "deferred_period": null, + "atm_transfer_reference": { + "transaction_id": null, + "company_id": null + }, + "site_id": "MLB000009", + "payer_id": 200000010, + "order_id": 2000000000000011, + "currency_id": "BRL000012", + "status": "approved", + "transaction_order_id": null + } + ], + "fulfilled": true, + "taxes": { + "amount": null, + "currency_id": null, + "id": null + }, + "order_request": { + "change": null, + "return": null + }, + "expiration_date": "2026-09-01T06:40:51.000-04:00", + "feedback": { + "buyer": null, + "seller": null + }, + "shipping": { + "id": 20000000013 + }, + "date_closed": "2026-08-04T06:40:51.000-04:00", + "id": 2000000000000011, + "manufacturing_ending_date": null, + "order_items": [ + { + "item": { + "id": "MLB0000000014", + "title": "Texto inventado 2", + "category_id": "MLB000015", + "variation_id": null, + "seller_custom_field": "000016", + "global_price": null, + "net_weight": null, + "variation_attributes": [ + { + "name": "Texto inventado 17", + "id": "COLOR000018", + "value_id": "000019", + "value_name": "Texto inventado 20" + }, + { + "name": "Texto inventado 21", + "id": "FINISH000022", + "value_id": "0000023", + "value_name": "Texto inventado 24" + } + ], + "warranty": "Texto inventado 25", + "condition": "new", + "seller_sku": "000016" + }, + "quantity": 2, + "unit_price": 201.1, + "gross_price": 32.05, + "currency_id": "BRL000012", + "manufacturing_days": null, + "picked_quantity": null, + "requested_quantity": { + "measure": "unit", + "value": 2 + }, + "sale_fee": 208.45, + "listing_type_id": "GOLDSPECIAL000028", + "base_exchange_rate": null, + "base_currency_id": null, + "bundle": null, + "element_id": 31 + } + ], + "date_last_updated": "2026-09-01T10:54:52+00:00", + "last_updated": "2026-09-01T06:54:51.000-04:00", + "comment": null, + "pack_id": 2000000000000030, + "coupon": { + "amount": 6, + "id": null + }, + "shipping_cost": null, + "date_created": "2026-08-04T06:40:48.000-04:00", + "date_created_ttl": null, + "pickup_id": null, + "status_detail": null, + "tags": [ + "catalog", + "paid", + "delivered", + "pack_order" + ], + "static_tags": [ + "pack_order" + ], + "buyer": { + "id": 200000010, + "nickname": "Texto inventado 31" + }, + "seller": { + "id": 200000005, + "nickname": "Texto inventado 32" + }, + "total_amount": 32.05, + "paid_amount": 32.05, + "currency_id": "BRL000012", + "status": "paid", + "context": { + "application": null, + "product_id": null, + "channel": "marketplace", + "site": "MLB000009", + "flows": [ + "catalog" + ] + } + }, + { + "payments": [ + { + "reason": "Texto inventado 33", + "status_code": null, + "total_paid_amount": 259.9, + "operation_type": "regular_payment", + "transaction_amount": 259.9, + "transaction_amount_refunded": 259.9, + "date_approved": "2026-08-18T07:59:23.000-04:00", + "collector": { + "id": 200000005 + }, + "coupon_id": null, + "installments": 1, + "authorization_code": "Texto inventado 35", + "taxes_amount": 6, + "id": 200000000036, + "date_last_modified": "2026-09-04T02:09:53.000-04:00", + "coupon_amount": 6, + "available_actions": [ + "Texto inventado 1" + ], + "shipping_cost": 6, + "installment_amount": 259.9, + "date_created": "2026-08-18T07:59:21.000-04:00", + "activation_uri": null, + "overpaid_amount": 6, + "card_id": 2000000037, + "status_detail": "bpp_refunded", + "issuer_id": "000038", + "payment_method_id": "MASTER000039", + "payment_type": "credit_card", + "deferred_period": null, + "atm_transfer_reference": { + "transaction_id": null, + "company_id": null + }, + "site_id": "MLB000009", + "payer_id": 2000000040, + "order_id": 2000000000000041, + "currency_id": "BRL000012", + "status": "refunded", + "transaction_order_id": null + } + ], + "fulfilled": false, + "taxes": { + "amount": null, + "currency_id": null, + "id": null + }, + "order_request": { + "change": null, + "return": null + }, + "expiration_date": "2026-09-15T08:00:00.000-04:00", + "feedback": { + "buyer": null, + "seller": null + }, + "shipping": { + "id": 20000000042 + }, + "date_closed": "2026-08-18T08:00:00.000-04:00", + "id": 2000000000000041, + "manufacturing_ending_date": null, + "order_items": [ + { + "item": { + "id": "MLB0000000043", + "title": "Texto inventado 33", + "category_id": "MLB000044", + "variation_id": null, + "seller_custom_field": "000045", + "global_price": null, + "net_weight": null, + "variation_attributes": [ + { + "name": "Texto inventado 17", + "id": "COLOR000018", + "value_id": "0000046", + "value_name": "Texto inventado 47" + }, + { + "name": "Texto inventado 21", + "id": "FINISH000022", + "value_id": "000048", + "value_name": "Texto inventado 49" + } + ], + "warranty": "Texto inventado 25", + "condition": "new", + "seller_sku": null + }, + "quantity": 1, + "unit_price": 259.9, + "gross_price": 259.9, + "currency_id": "BRL000012", + "manufacturing_days": null, + "picked_quantity": null, + "requested_quantity": { + "measure": "unit", + "value": 1 + }, + "sale_fee": 377.5, + "listing_type_id": "GOLDSPECIAL000028", + "base_exchange_rate": null, + "base_currency_id": null, + "bundle": null, + "element_id": 31 + } + ], + "date_last_updated": "2026-09-06T06:21:43+00:00", + "last_updated": "2026-09-04T02:09:54.000-04:00", + "comment": null, + "pack_id": 2000000000000051, + "coupon": { + "amount": 6, + "id": null + }, + "shipping_cost": null, + "date_created": "2026-08-18T07:59:20.000-04:00", + "date_created_ttl": null, + "pickup_id": null, + "status_detail": null, + "tags": [ + "catalog", + "delivered", + "not_paid", + "d2c", + "one_shot", + "pack_order" + ], + "static_tags": [ + "d2c", + "one_shot", + "pack_order" + ], + "buyer": { + "id": 2000000040, + "nickname": "Texto inventado 52" + }, + "seller": { + "id": 200000005, + "nickname": "Texto inventado 32" + }, + "total_amount": 259.9, + "paid_amount": 6, + "mediations": [ + { + "id": 2000000053 + } + ], + "currency_id": "BRL000012", + "status": "cancelled", + "cancel_detail": { + "group": "mediations", + "code": "mediations", + "description": "Texto inventado 54", + "requested_by": "meli", + "date": "2026-09-04T02:09:54.000-04:00", + "application_id": "0000000000000055" + }, + "context": { + "application": null, + "product_id": null, + "channel": "marketplace", + "site": "MLB000009", + "flows": [ + "catalog" + ] + } + } + ], + "sort": { + "id": "DATEASC000056", + "name": "Texto inventado 57" + }, + "available_sorts": [ + { + "id": "DATEDESC000058", + "name": "Texto inventado 59" + } + ], + "filters": [], + "paging": { + "total": 2, + "offset": 0, + "limit": 50 + }, + "display": "complete" + } +} diff --git a/packages/connectors/src/mercado-livre/recordings/perguntas-varredura.json b/packages/connectors/src/mercado-livre/recordings/perguntas-varredura.json new file mode 100644 index 00000000..8f6a7728 --- /dev/null +++ b/packages/connectors/src/mercado-livre/recordings/perguntas-varredura.json @@ -0,0 +1,117 @@ +{ + "service": "Mercado Livre", + "connector": "mercado-livre", + "case": "perguntas-varredura", + "route": { + "method": "GET", + "path": "/questions/search", + "query": [ + "api_version", + "limit", + "search_type", + "seller_id" + ] + }, + "recordedAt": "2026-11-12", + "status": 200, + "replaced": [ + "available_filters.id", + "available_filters.name", + "available_filters.values", + "filters.api_version", + "filters.caller", + "filters.search_type", + "filters.seller", + "questions.answer.date_created", + "questions.answer.status", + "questions.answer.text", + "questions.date_created", + "questions.from.id", + "questions.id", + "questions.item_id", + "questions.seller_id", + "questions.status", + "questions.text", + "scroll_id", + "seller" + ], + "body": { + "total": 1, + "limit": 50, + "questions": [ + { + "date_created": "2026-09-05T10:12:28.003-04:00", + "item_id": "MLB0000000001", + "seller_id": 200000002, + "status": "Texto inventado 3", + "text": "Texto inventado 4", + "tags": [], + "ai_categories": [], + "id": 20000000005, + "answer": { + "text": "Texto inventado 6", + "status": "Texto inventado 7", + "date_created": "2026-09-15T08:00:00.000-04:00" + }, + "from": { + "id": 200000008 + }, + "hold": false, + "deleted_from_listing": false + } + ], + "scroll_id": "YWNJZXNZX0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000009", + "filters": { + "limit": 50, + "offset": 0, + "api_version": "Texto inventado 10", + "is_admin": false, + "sorts": [], + "caller": 200000002, + "seller": 200000002, + "search_type": "Texto inventado 11" + }, + "available_filters": [ + { + "id": "ITEM000012", + "name": "Texto inventado 13", + "type": "text" + }, + { + "id": "FROM000014", + "name": "Texto inventado 15", + "type": "number" + }, + { + "id": "TOTALDIVISIONS000016", + "name": "Texto inventado 17", + "type": "number" + }, + { + "id": "DIVISION000018", + "name": "Texto inventado 19", + "type": "number" + }, + { + "id": "STATUS000020", + "name": "Texto inventado 21", + "type": "text", + "values": [ + "Texto inventado 3", + "Texto inventado 22", + "Texto inventado 23", + "Texto inventado 24", + "Texto inventado 25", + "Texto inventado 26", + "Texto inventado 27" + ] + } + ], + "available_sorts": [ + "item_id", + "from_id", + "date_created", + "seller_id" + ] + } +} diff --git a/scripts/record-connector-swap.ts b/scripts/record-connector-swap.ts index bdc50d68..918ec0a6 100644 --- a/scripts/record-connector-swap.ts +++ b/scripts/record-connector-swap.ts @@ -12,14 +12,14 @@ const ANCHOR = Date.parse('2026-09-15T12:00:00.000Z') const INSTANT = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?(Z|[+-]\d{2}:\d{2})$/ /** Keys whose number is an identity or an amount: replaced. Paging counts and quantities stay. */ -const NUMBER_KEY = /(^|_)id$|^id$|seller|buyer|number|price|amount|cost|fee|tax/i +const NUMBER_KEY = /(^|_)id$|^id$|seller|buyer|caller|number|price|amount|cost|fee|tax/i /** Keys whose text is a person's, a seller's or a listing's own words: always replaced, whatever it looks like. */ const TEXT_KEY = /name|nick|title|text|email|phone|address|street|city|zip|permalink|thumbnail|picture|sku|seller_custom_field|document|number|comment|description|identification/i /** Keys whose text names a state of the service, not a person or a thing: kept when it looks like a code. */ const CODE = /^[a-z][a-z0-9_]{0,40}$/ /** Keys whose text is an identity: replaced by one of the same shape. */ -const ID_KEY = /(^|_)id$|^id$|scroll|seller|buyer/i +const ID_KEY = /(^|_)id$|^id$|scroll|seller|buyer|caller/i export type Swapped = Readonly<{ body: unknown; replaced: readonly string[]; path: string }> diff --git a/scripts/record-connector.test.ts b/scripts/record-connector.test.ts index a2d408bd..d66d4cf0 100644 --- a/scripts/record-connector.test.ts +++ b/scripts/record-connector.test.ts @@ -151,6 +151,21 @@ test('o mesmo valor real vira o mesmo inventado, e a ordem das datas fica', () = ) }) +test('o id da conta que chama, ecoado nos filtros, vira o mesmo inventado do vendedor', () => { + const done = swapped({ + body: { seller_id: 987654321, filters: { caller: 987654321, seller: 987654321, limit: 50 } }, + account: '987654321', + path: '/questions/search', + }) + assert.equal(JSON.stringify(done.body).includes('987654321'), false) + const body = z + .object({ seller_id: z.number(), filters: z.object({ caller: z.number(), limit: z.number() }) }) + .parse(done.body) + assert.equal(body.filters.caller, body.seller_id) + assert.equal(body.filters.limit, 50) + assert.ok(done.replaced.includes('filters.caller')) +}) + test('um endereço ou prefixo de credencial que sobra recusa a gravação', () => { assert.throws( () => swapped({ body: { 'maria@exemplo.test': 'x' }, account: SELLER, path: '/x' }), From 204ee4afd1d0dcd510f96f98be26c5680d3cdb04 Mon Sep 17 00:00:00 2001 From: developmentconexus Date: Wed, 11 Nov 2026 22:13:40 -0300 Subject: [PATCH 6/8] =?UTF-8?q?feat(connectors):=20o=20dubl=C3=AA=20do=20M?= =?UTF-8?q?ercado=20Livre=20responde=20pelas=20formas=20gravadas,=20a=20co?= =?UTF-8?q?nformidade=20da=20lista=20anda=20por=20um=20campo=20escalar,=20?= =?UTF-8?q?e=20a=20varredura=20recusa=20offset=20sem=20o=20probe=20mandar?= =?UTF-8?q?=20um?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- biome.json | 1 + .../connectors/src/kernel/conformance-list.ts | 6 +- .../connectors/src/mercado-livre/fake-data.ts | 21 +- .../src/mercado-livre/fake-routes-more.ts | 37 +-- .../src/mercado-livre/fake-routes.ts | 10 +- .../connectors/src/mercado-livre/fake.test.ts | 85 +++++- .../src/mercado-livre/operations-list.test.ts | 285 ++++++++++++++++++ .../src/mercado-livre/operations.test.ts | 252 ---------------- .../src/mercado-livre/operations/anuncios.ts | 2 +- .../src/mercado-livre/operations/perguntas.ts | 2 +- .../src/mercado-livre/recordings.ts | 51 ++++ 11 files changed, 468 insertions(+), 284 deletions(-) create mode 100644 packages/connectors/src/mercado-livre/operations-list.test.ts create mode 100644 packages/connectors/src/mercado-livre/recordings.ts diff --git a/biome.json b/biome.json index 67976bbe..e7fe7f34 100644 --- a/biome.json +++ b/biome.json @@ -13,6 +13,7 @@ "!apps/web/src/routeTree.gen.ts", "!apps/web/src/builder/model-accounts/marks", "!packages/connectors/src/*/logo.svg", + "!packages/connectors/src/*/recordings", "!packages/app/starter", "!packages/app/skills" ] diff --git a/packages/connectors/src/kernel/conformance-list.ts b/packages/connectors/src/kernel/conformance-list.ts index aba10013..63adee74 100644 --- a/packages/connectors/src/kernel/conformance-list.ts +++ b/packages/connectors/src/kernel/conformance-list.ts @@ -11,7 +11,7 @@ import type { ListRead, Operation } from './operation.ts' import { listInputNames, valueAt } from './operation.ts' import { Rows } from './rows.ts' -/** The listing's reading by the fixed input of a probe: the paging and changed-since names leave it, and the copy takes the rest, keyed by `key`, a field the double's rows have. */ +/** The listing's reading by the fixed input of a probe: the paging and changed-since names leave it, and the copy takes the rest, keyed by `key`, a scalar field the double's rows have. */ function walkDefinition( operation: Operation, list: ListRead, @@ -84,7 +84,9 @@ async function walkList( }>, ): Promise { const { operation, list, first } = a - const key = Object.keys(first.linhas[0] ?? {})[0] + const key = Object.entries(first.linhas[0] ?? {}).find( + ([, value]) => typeof value === 'string' || typeof value === 'number', + )?.[0] if (key === undefined) { assert.equal(first.total ?? 0, 0, `${operation.id} lacks rows the total counts`) return diff --git a/packages/connectors/src/mercado-livre/fake-data.ts b/packages/connectors/src/mercado-livre/fake-data.ts index 99039e83..bd02d73e 100644 --- a/packages/connectors/src/mercado-livre/fake-data.ts +++ b/packages/connectors/src/mercado-livre/fake-data.ts @@ -1,3 +1,5 @@ +import { rowsOf } from './recordings.ts' + // Invented values only (S40). The buyer's name, e-mail, phone, address and CPF are here on // purpose: the kit must drop them (`operations.test.ts`, "sem dados do comprador"). @@ -50,12 +52,19 @@ export const ORDER_BODY = { ], } -/** 120 orders, the first `ORDER_BODY`; each changed a minute after the one before, so a changed-since filter keeps the newest. */ -export const ORDERS = Array.from({ length: 120 }, (_, index) => ({ - ...ORDER_BODY, - id: ORDER + index, - date_last_updated: new Date(Date.UTC(2026, 8, 10, 13, index)).toISOString(), -})) +let ordersKept: readonly (Record & { date_last_updated: string })[] | undefined + +/** 120 orders, each shaped as a recorded one with the planted buyer; each changed a minute after the one before, so a changed-since filter keeps the newest. */ +export function ordersOf() { + const recorded = rowsOf('pedidos-alterados', 'results') + ordersKept ??= Array.from({ length: 120 }, (_, index) => ({ + ...recorded[index % recorded.length], + buyer: BUYER, + id: ORDER + index, + date_last_updated: new Date(Date.UTC(2026, 8, 10, 13, index)).toISOString(), + })) + return ordersKept +} /** The rows of a scan's page: its `scroll_id` carries where the next page starts (`-`). */ export function scanOf(rows: readonly T[], query: URLSearchParams, route: string) { diff --git a/packages/connectors/src/mercado-livre/fake-routes-more.ts b/packages/connectors/src/mercado-livre/fake-routes-more.ts index 65b30129..648dd62a 100644 --- a/packages/connectors/src/mercado-livre/fake-routes-more.ts +++ b/packages/connectors/src/mercado-livre/fake-routes-more.ts @@ -13,22 +13,24 @@ import { SELLER_PATH, scanOf, } from './fake-data.ts' +import { envelope, rowsOf } from './recordings.ts' -const QUESTIONS = Array.from({ length: 12 }, (_, index) => ({ - id: 9000000001 + index, - text: `Tem na cor azul? (${index + 1})`, - status: 'UNANSWERED', - date_created: '2026-09-11T09:00:00.000-03:00', - item_id: ITEMS[0], - seller_id: Number(SELLER.ref), - from: { - id: BUYER.id, - answered_questions: 3, - nickname: BUYER.nickname, - email: BUYER.email, - }, - answer: null, -})) +let questionsKept: readonly Record[] | undefined + +/** 12 questions shaped as the recorded one, every fourth answered as it was; the asker's name and e-mail are planted for the kit to drop. */ +function questionsOf() { + const recorded = rowsOf('perguntas-varredura', 'questions') + questionsKept ??= Array.from({ length: 12 }, (_, index) => ({ + ...recorded[0], + id: 9000000001 + index, + text: `Tem na cor azul? (${index + 1})`, + ...(index % 4 === 3 ? {} : { status: 'UNANSWERED', answer: null }), + item_id: ITEMS[0], + seller_id: Number(SELLER.ref), + from: { id: BUYER.id, answered_questions: 3, nickname: BUYER.nickname, email: BUYER.email }, + })) + return questionsKept +} const PERIODS = Array.from({ length: 14 }, (_, index) => { const month = String(index + 1).padStart(2, '0') @@ -55,9 +57,10 @@ export const MORE_ROUTES: readonly Route[] = [ answer: ({ query }) => { if (query.get('search_type') === 'scan') { if (badScroll(query, 'questions')) return BAD_SCROLL - const scan = scanOf(QUESTIONS, query, 'questions') + const scan = scanOf(questionsOf(), query, 'questions') return { body: { + ...envelope('perguntas-varredura', 'questions'), total: scan.total, limit: scan.limit, questions: scan.rows, @@ -65,7 +68,7 @@ export const MORE_ROUTES: readonly Route[] = [ }, } } - const page = pageOf(QUESTIONS, query, 50) + const page = pageOf(questionsOf(), query, 50) return { body: { total: page.total, diff --git a/packages/connectors/src/mercado-livre/fake-routes.ts b/packages/connectors/src/mercado-livre/fake-routes.ts index d3f9cf23..ec718f47 100644 --- a/packages/connectors/src/mercado-livre/fake-routes.ts +++ b/packages/connectors/src/mercado-livre/fake-routes.ts @@ -7,7 +7,7 @@ import { itemBody, ORDER, ORDER_BODY, - ORDERS, + ordersOf, pageOf, paging, SELLER, @@ -16,6 +16,7 @@ import { scanOf, } from './fake-data.ts' import { MORE_ROUTES } from './fake-routes-more.ts' +import { envelope } from './recordings.ts' /** The routes of the double, by permission group; the order is the order they are tried. */ export const ROUTES: readonly Route[] = [ @@ -29,6 +30,7 @@ export const ROUTES: readonly Route[] = [ const scan = scanOf(ITEMS, query, 'items') return { body: { + ...envelope('anuncios-varredura-1', 'results'), seller_id: SELLER.ref, results: scan.rows, scroll_id: scan.scrollId, @@ -104,14 +106,14 @@ export const ROUTES: readonly Route[] = [ const from = query.get('order.date_last_updated.from') const changed = from === null - ? ORDERS - : ORDERS.filter((order) => Date.parse(order.date_last_updated) >= Date.parse(from)) + ? ordersOf() + : ordersOf().filter((order) => Date.parse(order.date_last_updated) >= Date.parse(from)) const page = pageOf(changed, query, 50) return { body: { + ...envelope('pedidos-alterados', 'results'), results: page.rows, paging: paging(page.total, query), - sort: { id: 'date_desc' }, }, } }, diff --git a/packages/connectors/src/mercado-livre/fake.test.ts b/packages/connectors/src/mercado-livre/fake.test.ts index 7b8cf6ef..fe59d50f 100644 --- a/packages/connectors/src/mercado-livre/fake.test.ts +++ b/packages/connectors/src/mercado-livre/fake.test.ts @@ -1,9 +1,16 @@ import assert from 'node:assert/strict' import { test } from 'node:test' +import { z } from 'zod' +import { SENT_AT } from '../kernel/conformance-declared.ts' +import { answerOf } from '../kernel/conformance-session.ts' import { refreshRequest } from '../kernel/oauth.ts' +import { requestOf } from '../kernel/operation.ts' +import { toRequest } from '../kernel/request.ts' import { mercadoLivre } from './definition.ts' import { mercadoLivreFake } from './fake.ts' -import { authorize, getWith, grant } from './test-session.ts' +import type { Case } from './recordings.ts' +import { recording } from './recordings.ts' +import { ACCOUNT, authorize, getWith, grant } from './test-session.ts' const auth = mercadoLivre.auth @@ -93,3 +100,79 @@ test('expireAccess dá 401 nas chaves de acesso emitidas, e uma renovação dá if (renewed.kind !== 'OK') return assert.fail('renewal') assert.equal(await read(renewed.accessToken), 200) }) + +/** Each path of a body (`results[].buyer.id`) with the JSON types seen there, across every element of an array. */ +function shapeOf(value: unknown, at = '', found = new Map>()) { + const type = value === null ? 'null' : Array.isArray(value) ? 'array' : typeof value + found.set(at, (found.get(at) ?? new Set()).add(type)) + if (Array.isArray(value)) for (const item of value) shapeOf(item, `${at}[]`, found) + else if (typeof value === 'object' && value !== null) + for (const [key, item] of Object.entries(value)) shapeOf(item, `${at}.${key}`, found) + return found +} + +/** What the double plants for the kit to drop (`operations.test.ts`, "sem dados do comprador"): the service never sends it on these routes. */ +const PLANTED = ['.results[].buyer.', '.questions[].from.'] + +const SCAN = { scan: true } as const + +function bodyOf(answered: { body?: unknown }) { + return z.record(z.string(), z.unknown()).parse(answered.body) +} + +/** The route with the account's own id left out: the recording carries an invented one. */ +function accountless(path: string) { + return path.replace(/\/users\/[^/]+/, '/users/:account') +} + +test('as respostas gravadas: o dublê responde as quatro formas com as chaves, os tipos e a rota do serviço real', async () => { + const fake = mercadoLivreFake() + const granted = await authorize(fake) + if (granted.kind !== 'OK') return assert.fail('authorize') + const token = granted.accessToken + async function ask(id: string, input: Record) { + const operation = mercadoLivre.operations.find((each) => each.id === id) ?? assert.fail(id) + const request = requestOf(operation, input, ACCOUNT, SENT_AT) + const answered = await answerOf(await fake.handle(toRequest(mercadoLivre, request, token))) + return { request, answered } + } + const first = await ask('anuncios', { ...SCAN, limit: 100 }) + const scroll = z.string().parse(bodyOf(first.answered).scroll_id) + let last = first + for (let pages = 0; pages < 100; pages++) { + const results = z.array(z.unknown()).parse(bodyOf(last.answered).results) + if (results.length === 0 && pages > 0) break + last = await ask('anuncios', { + scrollId: z.string().parse(bodyOf(last.answered).scroll_id), + limit: 100, + }) + } + assert.notEqual(scroll, '') + const cases: [Case, Awaited>][] = [ + [ + 'pedidos-alterados', + await ask('pedidos', { offset: 0, limit: 50, changedFrom: '2026-09-10T00:00:00.000-03:00' }), + ], + ['anuncios-varredura-1', first], + ['anuncios-varredura-2', last], + ['perguntas-varredura', await ask('perguntas', { ...SCAN, limit: 50 })], + ] + for (const [name, { request, answered }] of cases) { + const real = recording(name) + assert.equal(accountless(request.path), accountless(real.route.path), `${name}: a rota`) + assert.deepEqual(Object.keys(request.query).sort(), real.route.query, `${name}: o pedido`) + assert.equal(answered.status, real.status, name) + const given = shapeOf(bodyOf(answered)) + const recorded = shapeOf(real.body) + for (const [at, types] of recorded) + for (const type of types) + assert.ok(given.get(at)?.has(type), `${name}: o dublê não manda ${at} como ${type}`) + for (const at of given.keys()) + assert.ok( + recorded.has(at) || PLANTED.some((planted) => at.startsWith(planted)), + `${name}: o serviço real não manda ${at}`, + ) + } + assert.deepEqual(bodyOf(last.answered).results, [], 'a varredura acaba numa página sem linhas') + assert.equal(typeof bodyOf(last.answered).scroll_id, 'string', '...que ainda traz o scroll_id') +}) diff --git a/packages/connectors/src/mercado-livre/operations-list.test.ts b/packages/connectors/src/mercado-livre/operations-list.test.ts new file mode 100644 index 00000000..85eb3c2d --- /dev/null +++ b/packages/connectors/src/mercado-livre/operations-list.test.ts @@ -0,0 +1,285 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { z } from 'zod' +import { SENT_AT } from '../kernel/conformance-declared.ts' +import { answerOf } from '../kernel/conformance-session.ts' +import type { CopyDefinitionV2, CopyPosition } from '../kernel/copy.ts' +import { CopyDefinitionV2 as CopyDefinitionV2Schema } from '../kernel/copy.ts' +import { listInputOf, listPageOf } from '../kernel/list.ts' +import type { Operation } from '../kernel/operation.ts' +import { requestOf } from '../kernel/operation.ts' +import { toRequest } from '../kernel/request.ts' +import { Rows } from '../kernel/rows.ts' +import { mercadoLivre } from './definition.ts' +import { mercadoLivreFake } from './fake.ts' +import { mensagensDoPacote } from './operations/mensagens.ts' +import { recording } from './recordings.ts' +import { ACCOUNT, authorize } from './test-session.ts' + +function scanned(results: unknown[], extra: Record) { + return operationOf('anuncios').answer({ status: 200, body: { results, ...extra } }) +} + +function operationOf(id: string) { + const found = mercadoLivre.operations.find((operation) => operation.id === id) + assert.ok(found, id) + return found +} + +test('pedidos pelo que mudou', () => { + const pedidos = operationOf('pedidos') + const changedFrom = '2026-11-09T10:00:00.000-03:00' + const query = requestOf(pedidos, { offset: 0, limit: 50, changedFrom }, ACCOUNT, SENT_AT).query + assert.equal(query['order.date_last_updated.from'], changedFrom) + assert.equal(query.offset, '0') + assert.equal(query.limit, '50') + assert.equal( + 'order.date_last_updated.from' in + requestOf(pedidos, { offset: 0, limit: 5 }, ACCOUNT, SENT_AT).query, + false, + 'sem a entrada, a leitura de hoje sai igual', + ) + assert.equal( + pedidos.input.safeParse({ offset: 0, limit: 5, changedFrom: '2026-11-09' }).success, + false, + ) +}) + +test('a varredura: a primeira página manda scan, as seguintes o scroll_id', () => { + for (const id of ['anuncios', 'perguntas']) { + const read = operationOf(id) + const first = requestOf(read, { scan: true, limit: 50 }, ACCOUNT, SENT_AT).query + assert.equal(first.search_type, 'scan', id) + assert.equal(first.scroll_id, undefined, id) + assert.equal(first.offset, undefined, id) + const next = requestOf(read, { scrollId: 'abc', limit: 50 }, ACCOUNT, SENT_AT).query + assert.equal(next.scroll_id, 'abc', id) + assert.equal(next.search_type, 'scan', id) + assert.equal( + 'search_type' in requestOf(read, { offset: 0, limit: 50 }, ACCOUNT, SENT_AT).query, + false, + `${id}: por deslocamento, como hoje`, + ) + assert.equal(read.input.safeParse({ scan: false, limit: 50 }).success, false, id) + for (const extra of [{ scan: true }, { scrollId: 'abc' }]) + assert.equal(read.input.safeParse({ ...extra, offset: 0, limit: 50 }).success, false, id) + assert.equal(read.input.safeParse({ scrollId: '', limit: 50 }).success, false, id) + } +}) + +test('o cursor da resposta gravada de uma varredura vai em seguinte, e a página sem linhas é o fim', () => { + const page = recording('anuncios-varredura-1').body + const ids = z.array(z.string()).parse(page.results) + assert.deepEqual(operationOf('anuncios').answer({ status: 200, body: page }), { + kind: 'OK', + value: { + linhas: ids.map((id) => ({ id })), + temMais: true, + seguinte: z.string().parse(page.scroll_id), + total: ids.length, + }, + }) + assert.deepEqual( + operationOf('anuncios').answer({ status: 200, body: recording('anuncios-varredura-2').body }), + { kind: 'OK', value: { linhas: [], temMais: false, total: ids.length } }, + 'a página sem linhas traz o scroll_id e é o fim', + ) + const questions = recording('perguntas-varredura').body + const asked = operationOf('perguntas').answer({ status: 200, body: questions }) + assert.equal(asked.kind, 'OK') + if (asked.kind !== 'OK') return + const { linhas, ...rest } = Rows.parse(asked.value) + assert.equal(linhas.length, 1) + assert.deepEqual(rest, { + temMais: true, + seguinte: z.string().parse(questions.scroll_id), + total: z.number().parse(questions.total), + }) + assert.deepEqual(scanned(['a'], { paging: { total: 1, offset: 0, limit: 5 } }), { + kind: 'OK', + value: { linhas: [{ id: 'a' }], temMais: false, total: 1 }, + }) +}) + +test('o total e o temMais saem das mesmas contagens: sem deslocamento só o total, e um total quebrado nenhum dos dois', () => { + assert.deepEqual(scanned(['a'], { paging: { total: 9 } }), { + kind: 'OK', + value: { linhas: [{ id: 'a' }], temMais: null, total: 9 }, + }) + assert.deepEqual(scanned(['a'], { paging: { total: 9.5, offset: 0, limit: 1 } }), { + kind: 'OK', + value: { linhas: [{ id: 'a' }], temMais: null }, + }) +}) + +/** Every page of a list read of the double, through the copy's own input and page: the cells of each row. */ +async function walked( + operation: Operation, + definition: CopyDefinitionV2, + since: string | null = null, +) { + const list = operation.list ?? assert.fail(`${operation.id} declares no list`) + const fake = mercadoLivreFake() + const granted = await authorize(fake) + if (granted.kind !== 'OK') return assert.fail('authorize') + const cells: (string | null)[][] = [] + let position: CopyPosition | null = null + for (let pages = 0; pages < 100; pages++) { + const input = operation.input.parse(listInputOf({ list, definition, position, since })) + const request = requestOf(operation, input, ACCOUNT, SENT_AT) + const answered = operation.answer( + await answerOf(await fake.handle(toRequest(mercadoLivre, request, granted.accessToken))), + ) + if (answered.kind !== 'OK') return assert.fail(`${operation.id}: ${answered.kind}`) + const rows = Rows.parse(answered.value) + const page = listPageOf({ list, definition, position, rows, at: SENT_AT }) + if (page.kind !== 'PAGE') return assert.fail(`${operation.id}: ${page.kind}`) + cells.push(...page.rows.map((row) => [...row])) + if (!page.more) return { cells, pages: pages + 1 } + position = page.next + } + return assert.fail(`${operation.id} never ends`) +} + +function definitionOf( + operation: Operation, + columns: { name: string; type: string; path: string }[], + changed: string | null = null, +) { + return CopyDefinitionV2Schema.parse({ + version: 2, + source: { kind: 'OPERATION', operation: operation.id, input: {} }, + description: 'Teste.', + columns, + key: [columns[0]?.name], + changedSince: changed === null ? null : { column: changed }, + }) +} + +test('pedidos declara a lista por deslocamento e o filtro do que mudou', async () => { + const pedidos = operationOf('pedidos') + assert.deepEqual(pedidos.list, { + paging: { kind: 'OFFSET', offset: 'offset', limit: 'limit', max: 50, ceiling: null }, + changed: { input: 'changedFrom', field: 'date_last_updated' }, + }) + const definition = definitionOf( + pedidos, + [ + { name: 'id', type: 'bigint', path: 'id' }, + { name: 'alterado', type: 'timestamp', path: 'date_last_updated' }, + ], + 'alterado', + ) + const all = await walked(pedidos, definition) + assert.equal(all.cells.length, 120, 'os 120 pedidos do dublê') + assert.equal(all.pages, 3) + const changed = await walked(pedidos, definition, '2026-09-10T14:50:00.000Z') + assert.deepEqual( + changed.cells.map((row) => row[1]), + all.cells + .map((row) => row[1]) + .filter((at) => Date.parse(at ?? '') >= Date.parse('2026-09-10T14:50:00.000Z')), + 'só o que mudou desde o instante', + ) + assert.equal(changed.cells.length, 10) +}) + +test('a varredura passa de mil', async () => { + for (const [id, max] of [ + ['anuncios', 100], + ['perguntas', 50], + ] as const) { + assert.deepEqual( + operationOf(id).list, + { + paging: { + kind: 'CURSOR', + cursor: 'scrollId', + limit: 'limit', + max, + start: { scan: true }, + ttlMs: 300_000, + }, + changed: null, + }, + id, + ) + } + const anuncios = operationOf('anuncios') + const all = await walked( + anuncios, + definitionOf(anuncios, [{ name: 'id', type: 'text', path: 'id' }]), + ) + assert.equal(all.cells.length, 1300, 'os 1.300 anúncios, passando de mil') + assert.equal(new Set(all.cells.map((row) => row[0])).size, 1300, 'nenhum repete') + const list = anuncios.list ?? assert.fail('list') + const definition = definitionOf(anuncios, [{ name: 'id', type: 'text', path: 'id' }]) + function pageOf(body: Record) { + const answered = scanned(['MLB1'], body) + if (answered.kind !== 'OK') return assert.fail(answered.kind) + return listPageOf({ + list, + definition, + position: null, + rows: Rows.parse(answered.value), + at: SENT_AT, + }) + } + assert.deepEqual( + pageOf({ paging: { total: 9, offset: 0, limit: 1 } }), + { kind: 'UNREADABLE', reason: 'NO_CURSOR' }, + 'linhas, mais por vir e sem scroll_id', + ) + assert.deepEqual( + pageOf({ paging: { total: 9, limit: 1 } }), + { kind: 'UNREADABLE', reason: 'NO_MORE_SIGNAL' }, + 'linhas sem scroll_id e sem deslocamento: nenhum sinal de fim', + ) +}) + +test('as outras listas por deslocamento, e as mensagens de um pacote sem lista', () => { + for (const [id, max] of [ + ['reclamacoes', 30], + ['periodosDeFaturamento', 12], + ] as const) + assert.deepEqual( + operationOf(id).list, + { + paging: { kind: 'OFFSET', offset: 'offset', limit: 'limit', max, ceiling: null }, + changed: null, + }, + id, + ) + assert.equal(mensagensDoPacote.list, undefined, 'pede o packId de um pacote') + const listed = mercadoLivre.operations.filter((operation) => operation.list !== undefined) + assert.deepEqual(listed.map((operation) => operation.id).sort(), [ + 'anuncios', + 'pedidos', + 'perguntas', + 'periodosDeFaturamento', + 'reclamacoes', + ]) +}) + +test('o comprador não chega à cópia', async () => { + const personal = ['maria.ficticia', '988776655', '52998224725', 'Fictícia', 'COMPRADORFICTICIO'] + for (const id of ['pedidos', 'perguntas']) { + const operation = operationOf(id) + const fields = ['id', 'buyer', 'from', 'status', 'order_items', 'text', 'date_created'] + const columns = [ + { name: 'id', type: 'bigint', path: 'id' }, + ...fields.slice(1).map((field) => ({ name: field, type: 'json', path: field })), + ] + const all = await walked(operation, definitionOf(operation, columns)) + assert.ok(all.cells.length > 0, id) + const text = JSON.stringify(all.cells) + for (const value of personal) assert.ok(!text.includes(value), `${id} copia ${value}`) + function at(name: string) { + return columns.findIndex((column) => column.name === name) + } + assert.ok( + all.cells.every((row) => row[at(id === 'pedidos' ? 'buyer' : 'from')] === null), + `${id}: o objeto cortado é nulo`, + ) + } +}) diff --git a/packages/connectors/src/mercado-livre/operations.test.ts b/packages/connectors/src/mercado-livre/operations.test.ts index d639f134..ff36dc3f 100644 --- a/packages/connectors/src/mercado-livre/operations.test.ts +++ b/packages/connectors/src/mercado-livre/operations.test.ts @@ -2,13 +2,8 @@ import assert from 'node:assert/strict' import { test } from 'node:test' import { SENT_AT } from '../kernel/conformance-declared.ts' import { answerOf } from '../kernel/conformance-session.ts' -import type { CopyDefinitionV2, CopyPosition } from '../kernel/copy.ts' -import { CopyDefinitionV2 as CopyDefinitionV2Schema } from '../kernel/copy.ts' -import { listInputOf, listPageOf } from '../kernel/list.ts' -import type { Operation } from '../kernel/operation.ts' import { requestOf } from '../kernel/operation.ts' import { toRequest } from '../kernel/request.ts' -import { Rows } from '../kernel/rows.ts' import { mercadoLivre } from './definition.ts' import { mercadoLivreFake } from './fake.ts' import { mensagensDoPacote } from './operations/mensagens.ts' @@ -245,250 +240,3 @@ test('as visitas mandam as datas como a documentação mostra, só o dia', () => const probe = visitas.probe(new Map()) assert.ok(visitas.input.safeParse(probe).success) }) - -function scanned(results: unknown[], extra: Record) { - return operationOf('anuncios').answer({ status: 200, body: { results, ...extra } }) -} - -function operationOf(id: string) { - const found = mercadoLivre.operations.find((operation) => operation.id === id) - assert.ok(found, id) - return found -} - -test('pedidos pelo que mudou', () => { - const pedidos = operationOf('pedidos') - const changedFrom = '2026-11-09T10:00:00.000-03:00' - const query = requestOf(pedidos, { offset: 0, limit: 50, changedFrom }, ACCOUNT, SENT_AT).query - assert.equal(query['order.date_last_updated.from'], changedFrom) - assert.equal(query.offset, '0') - assert.equal(query.limit, '50') - assert.equal( - 'order.date_last_updated.from' in - requestOf(pedidos, { offset: 0, limit: 5 }, ACCOUNT, SENT_AT).query, - false, - 'sem a entrada, a leitura de hoje sai igual', - ) - assert.equal( - pedidos.input.safeParse({ offset: 0, limit: 5, changedFrom: '2026-11-09' }).success, - false, - ) -}) - -test('a varredura: a primeira página manda scan, as seguintes o scroll_id', () => { - for (const id of ['anuncios', 'perguntas']) { - const read = operationOf(id) - const first = requestOf(read, { scan: true, limit: 50 }, ACCOUNT, SENT_AT).query - assert.equal(first.search_type, 'scan', id) - assert.equal(first.scroll_id, undefined, id) - assert.equal(first.offset, undefined, id) - const next = requestOf(read, { scrollId: 'abc', limit: 50 }, ACCOUNT, SENT_AT).query - assert.equal(next.scroll_id, 'abc', id) - assert.equal(next.search_type, 'scan', id) - assert.equal( - 'search_type' in requestOf(read, { offset: 0, limit: 50 }, ACCOUNT, SENT_AT).query, - false, - `${id}: por deslocamento, como hoje`, - ) - assert.equal(read.input.safeParse({ scan: false, limit: 50 }).success, false, id) - for (const extra of [{ scan: true }, { scrollId: 'abc' }]) - assert.equal(read.input.safeParse({ ...extra, offset: 0, limit: 50 }).success, false, id) - assert.equal(read.input.safeParse({ scrollId: '', limit: 50 }).success, false, id) - } -}) - -test('o cursor da resposta de uma varredura vai em seguinte, e a página sem linhas é o fim', () => { - assert.deepEqual(scanned(['a', 'b'], { scroll_id: 'cursor-1', paging: { total: 9, limit: 2 } }), { - kind: 'OK', - value: { linhas: [{ id: 'a' }, { id: 'b' }], temMais: true, seguinte: 'cursor-1', total: 9 }, - }) - assert.deepEqual(scanned([], { scroll_id: 'cursor-2', paging: { total: 9, limit: 2 } }), { - kind: 'OK', - value: { linhas: [], temMais: false, total: 9 }, - }) - assert.deepEqual(scanned(['a'], { paging: { total: 1, offset: 0, limit: 5 } }), { - kind: 'OK', - value: { linhas: [{ id: 'a' }], temMais: false, total: 1 }, - }) -}) - -test('o total e o temMais saem das mesmas contagens: sem deslocamento só o total, e um total quebrado nenhum dos dois', () => { - assert.deepEqual(scanned(['a'], { paging: { total: 9 } }), { - kind: 'OK', - value: { linhas: [{ id: 'a' }], temMais: null, total: 9 }, - }) - assert.deepEqual(scanned(['a'], { paging: { total: 9.5, offset: 0, limit: 1 } }), { - kind: 'OK', - value: { linhas: [{ id: 'a' }], temMais: null }, - }) -}) - -/** Every page of a list read of the double, through the copy's own input and page: the cells of each row. */ -async function walked( - operation: Operation, - definition: CopyDefinitionV2, - since: string | null = null, -) { - const list = operation.list ?? assert.fail(`${operation.id} declares no list`) - const fake = mercadoLivreFake() - const granted = await authorize(fake) - if (granted.kind !== 'OK') return assert.fail('authorize') - const cells: (string | null)[][] = [] - let position: CopyPosition | null = null - for (let pages = 0; pages < 100; pages++) { - const input = operation.input.parse(listInputOf({ list, definition, position, since })) - const request = requestOf(operation, input, ACCOUNT, SENT_AT) - const answered = operation.answer( - await answerOf(await fake.handle(toRequest(mercadoLivre, request, granted.accessToken))), - ) - if (answered.kind !== 'OK') return assert.fail(`${operation.id}: ${answered.kind}`) - const rows = Rows.parse(answered.value) - const page = listPageOf({ list, definition, position, rows, at: SENT_AT }) - if (page.kind !== 'PAGE') return assert.fail(`${operation.id}: ${page.kind}`) - cells.push(...page.rows.map((row) => [...row])) - if (!page.more) return { cells, pages: pages + 1 } - position = page.next - } - return assert.fail(`${operation.id} never ends`) -} - -function definitionOf( - operation: Operation, - columns: { name: string; type: string; path: string }[], - changed: string | null = null, -) { - return CopyDefinitionV2Schema.parse({ - version: 2, - source: { kind: 'OPERATION', operation: operation.id, input: {} }, - description: 'Teste.', - columns, - key: [columns[0]?.name], - changedSince: changed === null ? null : { column: changed }, - }) -} - -test('pedidos declara a lista por deslocamento e o filtro do que mudou', async () => { - const pedidos = operationOf('pedidos') - assert.deepEqual(pedidos.list, { - paging: { kind: 'OFFSET', offset: 'offset', limit: 'limit', max: 50, ceiling: null }, - changed: { input: 'changedFrom', field: 'date_last_updated' }, - }) - const definition = definitionOf( - pedidos, - [ - { name: 'id', type: 'bigint', path: 'id' }, - { name: 'alterado', type: 'timestamp', path: 'date_last_updated' }, - ], - 'alterado', - ) - const all = await walked(pedidos, definition) - assert.equal(all.cells.length, 120, 'os 120 pedidos do dublê') - assert.equal(all.pages, 3) - const changed = await walked(pedidos, definition, '2026-09-10T14:50:00.000Z') - assert.deepEqual( - changed.cells.map((row) => row[1]), - all.cells - .map((row) => row[1]) - .filter((at) => Date.parse(at ?? '') >= Date.parse('2026-09-10T14:50:00.000Z')), - 'só o que mudou desde o instante', - ) - assert.equal(changed.cells.length, 10) -}) - -test('a varredura passa de mil', async () => { - for (const [id, max] of [ - ['anuncios', 100], - ['perguntas', 50], - ] as const) { - assert.deepEqual( - operationOf(id).list, - { - paging: { - kind: 'CURSOR', - cursor: 'scrollId', - limit: 'limit', - max, - start: { scan: true }, - ttlMs: 300_000, - }, - changed: null, - }, - id, - ) - } - const anuncios = operationOf('anuncios') - const all = await walked( - anuncios, - definitionOf(anuncios, [{ name: 'id', type: 'text', path: 'id' }]), - ) - assert.equal(all.cells.length, 1300, 'os 1.300 anúncios, passando de mil') - assert.equal(new Set(all.cells.map((row) => row[0])).size, 1300, 'nenhum repete') - const list = anuncios.list ?? assert.fail('list') - const definition = definitionOf(anuncios, [{ name: 'id', type: 'text', path: 'id' }]) - function pageOf(body: Record) { - const answered = scanned(['MLB1'], body) - if (answered.kind !== 'OK') return assert.fail(answered.kind) - return listPageOf({ - list, - definition, - position: null, - rows: Rows.parse(answered.value), - at: SENT_AT, - }) - } - assert.deepEqual( - pageOf({ paging: { total: 9, offset: 0, limit: 1 } }), - { kind: 'UNREADABLE', reason: 'NO_CURSOR' }, - 'linhas, mais por vir e sem scroll_id', - ) - assert.deepEqual( - pageOf({ paging: { total: 9, limit: 1 } }), - { kind: 'UNREADABLE', reason: 'NO_MORE_SIGNAL' }, - 'linhas sem scroll_id e sem deslocamento: nenhum sinal de fim', - ) -}) - -test('as outras listas por deslocamento, e as mensagens de um pacote sem lista', () => { - for (const [id, max] of [ - ['reclamacoes', 30], - ['periodosDeFaturamento', 12], - ] as const) - assert.deepEqual( - operationOf(id).list, - { - paging: { kind: 'OFFSET', offset: 'offset', limit: 'limit', max, ceiling: null }, - changed: null, - }, - id, - ) - assert.equal(mensagensDoPacote.list, undefined, 'pede o packId de um pacote') - const listed = mercadoLivre.operations.filter((operation) => operation.list !== undefined) - assert.deepEqual(listed.map((operation) => operation.id).sort(), [ - 'anuncios', - 'pedidos', - 'perguntas', - 'periodosDeFaturamento', - 'reclamacoes', - ]) -}) - -test('o comprador não chega à cópia', async () => { - const personal = ['maria.ficticia', '988776655', '52998224725', 'Fictícia', 'COMPRADORFICTICIO'] - for (const id of ['pedidos', 'perguntas']) { - const operation = operationOf(id) - const fields = ['id', 'buyer', 'from', 'status', 'order_items', 'text', 'date_created'] - const columns = [ - { name: 'id', type: 'bigint', path: 'id' }, - ...fields.slice(1).map((field) => ({ name: field, type: 'json', path: field })), - ] - const all = await walked(operation, definitionOf(operation, columns)) - assert.ok(all.cells.length > 0, id) - const text = JSON.stringify(all.cells) - for (const value of personal) assert.ok(!text.includes(value), `${id} copia ${value}`) - const at = (name: string) => columns.findIndex((column) => column.name === name) - assert.ok( - all.cells.every((row) => row[at(id === 'pedidos' ? 'buyer' : 'from')] === null), - `${id}: o objeto cortado é nulo`, - ) - } -}) diff --git a/packages/connectors/src/mercado-livre/operations/anuncios.ts b/packages/connectors/src/mercado-livre/operations/anuncios.ts index 0475d586..b08b67aa 100644 --- a/packages/connectors/src/mercado-livre/operations/anuncios.ts +++ b/packages/connectors/src/mercado-livre/operations/anuncios.ts @@ -40,7 +40,7 @@ export const anuncios = readOf({ }, changed: null, }, - probe: () => ({ offset: 0, limit: 5 }), + probe: () => ({ limit: 5 }), request: (input, account) => get(`/users/${sellerOf(account)}/items/search`, { status: input.status, diff --git a/packages/connectors/src/mercado-livre/operations/perguntas.ts b/packages/connectors/src/mercado-livre/operations/perguntas.ts index 39499f53..52836828 100644 --- a/packages/connectors/src/mercado-livre/operations/perguntas.ts +++ b/packages/connectors/src/mercado-livre/operations/perguntas.ts @@ -45,7 +45,7 @@ export const perguntas = readOf({ changed: null, }, omit: ['from'], // the asker: the question route adds their name and contact - probe: () => ({ offset: 0, limit: 5 }), + probe: () => ({ limit: 5 }), request: (input, account) => get('/questions/search', { seller_id: sellerOf(account), diff --git a/packages/connectors/src/mercado-livre/recordings.ts b/packages/connectors/src/mercado-livre/recordings.ts new file mode 100644 index 00000000..9d7369e7 --- /dev/null +++ b/packages/connectors/src/mercado-livre/recordings.ts @@ -0,0 +1,51 @@ +// What `scripts/record-connector.ts` saved, read by the double and by its conformance (testing T29): +// the four answers of the real service, every value invented, with the service, the route and the +// date beside the body. Read when asked for, never on import: the double ships with the Hub. +import { readFileSync } from 'node:fs' +import { join } from 'node:path' +import { z } from 'zod' + +const Recording = z.object({ + service: z.string().min(1), + connector: z.literal('mercado-livre'), + case: z.string().min(1), + route: z.object({ + method: z.literal('GET'), + path: z.string().startsWith('/'), + query: z.array(z.string()), + }), + recordedAt: z.iso.date(), + status: z.number().int(), + replaced: z.array(z.string()), + body: z.record(z.string(), z.unknown()), +}) +export type Recording = z.infer + +export type Case = + | 'pedidos-alterados' + | 'anuncios-varredura-1' + | 'anuncios-varredura-2' + | 'perguntas-varredura' + +const kept = new Map() + +export function recording(name: Case): Recording { + const found = kept.get(name) + if (found !== undefined) return found + const read = Recording.parse( + JSON.parse(readFileSync(join(import.meta.dirname, 'recordings', `${name}.json`), 'utf8')), + ) + kept.set(name, read) + return read +} + +/** The recorded body without the rows: what the service sends around them (filters, sorts, the account). */ +export function envelope(name: Case, rows: string): Record { + const { [rows]: _rows, ...rest } = recording(name).body + return rest +} + +/** The recorded rows under `key`, as the service sent them. */ +export function rowsOf(name: Case, key: string): readonly Record[] { + return z.array(z.record(z.string(), z.unknown())).parse(recording(name).body[key]) +} From 929e20cc434a1a7b48b58d91f833d8e01aafaedf Mon Sep 17 00:00:00 2001 From: developmentconexus Date: Wed, 11 Nov 2026 22:30:33 -0300 Subject: [PATCH 7/8] =?UTF-8?q?fix(connectors):=20o=20gravador=20deixa=20d?= =?UTF-8?q?e=20exportar=20inputOf,=20que=20s=C3=B3=20ele=20usa?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/record-connector-calls.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/record-connector-calls.ts b/scripts/record-connector-calls.ts index e1feac61..f03bd2d7 100644 --- a/scripts/record-connector-calls.ts +++ b/scripts/record-connector-calls.ts @@ -34,7 +34,7 @@ const PAGE_1 = 'anuncios-varredura-1' const WEEK_MS = 7 * 24 * 60 * 60 * 1000 /** The operation of a case and its input; the second listing page carries the first page's `scroll_id`. */ -export function inputOf( +function inputOf( name: Case, now: Date, first: unknown, From 1a87b7bca7f9a240aecf8599136abb51464b2684 Mon Sep 17 00:00:00 2001 From: developmentconexus Date: Wed, 11 Nov 2026 22:46:36 -0300 Subject: [PATCH 8/8] style(connectors): a assinatura de inputOf no formato do Biome --- scripts/record-connector-calls.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/scripts/record-connector-calls.ts b/scripts/record-connector-calls.ts index f03bd2d7..616e607f 100644 --- a/scripts/record-connector-calls.ts +++ b/scripts/record-connector-calls.ts @@ -34,11 +34,7 @@ const PAGE_1 = 'anuncios-varredura-1' const WEEK_MS = 7 * 24 * 60 * 60 * 1000 /** The operation of a case and its input; the second listing page carries the first page's `scroll_id`. */ -function inputOf( - name: Case, - now: Date, - first: unknown, -): { operation: string; input: unknown } { +function inputOf(name: Case, now: Date, first: unknown): { operation: string; input: unknown } { if (name === 'pedidos-alterados') { const week = new Date(now.getTime() - WEEK_MS) return {