diff --git a/generate-release-notes/generate-release-notes/lib/fetchGitHubCommits.js b/generate-release-notes/generate-release-notes/lib/fetchGitHubCommits.js index c62ee06..39e9229 100644 --- a/generate-release-notes/generate-release-notes/lib/fetchGitHubCommits.js +++ b/generate-release-notes/generate-release-notes/lib/fetchGitHubCommits.js @@ -9,11 +9,13 @@ export const fetchCommits = async ({ let nextPage = true; let commits = []; - console.log(`Fetching commits for ${repository} between ${baseTag} and ${tag}...`); + const token = process.env.GITHUB_TOKEN; + const headers = token ? { Authorization: `Bearer ${token}` } : {}; + console.log(`Fetching commits for ${repository} between ${baseTag} and ${tag} (${token ? 'authenticated' : 'unauthenticated'})...`); while (nextPage) { const githubQuery = `https://api.github.com/repos/${repository}/compare/${baseTag}...${tag}?per_page=${pageSize}&page=${page}`; console.log(`Fetching commits from ${githubQuery}`); - const githubResponse = await fetch(githubQuery); + const githubResponse = await fetch(githubQuery, { headers }); if (!githubResponse.ok) { throw new Error(`Failed to fetch commits from ${githubQuery}, status: ${githubResponse.status}`); } diff --git a/generate-release-notes/generate-release-notes/lib/fetchJiraIssues.js b/generate-release-notes/generate-release-notes/lib/fetchJiraIssues.js index 7ec3124..b85bbf7 100644 --- a/generate-release-notes/generate-release-notes/lib/fetchJiraIssues.js +++ b/generate-release-notes/generate-release-notes/lib/fetchJiraIssues.js @@ -1,22 +1,78 @@ // Description: Fetches the JIRA issues from bugs.openjdk.org -export default async function fetchReleaseNotes(version) { + +const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +async function fetchJiraJson(url, maxRetries = 5) { + const headers = { + Accept: 'application/json', + 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36', + }; + + for (let attempt = 1; attempt <= maxRetries; attempt += 1) { + try { + const response = await fetch(url, { headers, signal: AbortSignal.timeout(60000) }); + if (!response.ok) { + if (response.status === 429 || response.status >= 500) { + if (attempt < maxRetries) { + const backoff = Math.min(attempt * 10000, 60000); + console.warn(`Jira API HTTP ${response.status}. Retrying in ${backoff / 1000}s (attempt ${attempt}/${maxRetries})...`); + await delay(backoff); + continue; + } + } + throw new Error(`Jira API request failed: HTTP ${response.status} for ${url}`); + } + const contentType = response.headers.get('content-type') || ''; + if (!contentType.includes('application/json')) { + const body = await response.text(); + if (attempt < maxRetries) { + const backoff = Math.min(attempt * 10000, 60000); + console.warn(`Jira API returned non-JSON response (content-type: ${contentType}). Retrying in ${backoff / 1000}s (attempt ${attempt}/${maxRetries})...`); + await delay(backoff); + continue; + } + throw new Error(`Jira API returned non-JSON response (content-type: ${contentType}) for ${url}. Body starts with: ${body.slice(0, 200)}`); + } + return await response.json(); + } catch (err) { + if (attempt < maxRetries && (err.name === 'TimeoutError' || err.name === 'AbortError' || err.code === 'UND_ERR_CONNECT_TIMEOUT')) { + const backoff = Math.min(attempt * 10000, 60000); + console.warn(`Jira API request timed out. Retrying in ${backoff / 1000}s (attempt ${attempt}/${maxRetries})...`); + await delay(backoff); + continue; + } + throw err; + } + } +} + +export default async function fetchReleaseNotes(version, maxRetries = 5) { // fetch the release notes from the bugs.openjdk.org - const baseUrl = 'https://bugs.openjdk.java.net/rest/api/2/search?jql='; + const baseUrl = 'https://bugs.openjdk.org/rest/api/2/search?jql='; const jql = `project=JDK AND (status in (Closed, Resolved)) AND (resolution not in ("Won't Fix", "Duplicate", "Cannot Reproduce", "Not an Issue", "Withdrawn")) AND (labels not in (release-note, openjdk-na) OR labels is EMPTY) AND (summary !~ "release note") AND (issuetype != CSR) AND (fixVersion in (${version}))`; + const encodedJql = encodeURIComponent(jql); + const fields = 'summary,priority,components,customfield_10008,issuetype,issuelinks'; + const pageSize = 100; + const pageDelayMs = 2500; + // execute the initial fetch to get the total number of issues - const totalQuery = await fetch(`${baseUrl + jql}&startAt=1&maxResults=1`); - const initialRes = await totalQuery.json(); + const initialRes = await fetchJiraJson(`${baseUrl + encodedJql}&startAt=0&maxResults=1&fields=summary`, maxRetries); const { total } = initialRes; + console.log(`Found ${total} JIRA issues for JDK ${version}. Fetching in pages of ${pageSize}...`); + const JIRA_ISSUES = []; // fetch all the issues by page - for (let startAt = 0; startAt <= total + 50; startAt += 50) { - const query = await fetch(`${baseUrl + jql}&startAt=${startAt}&maxResults=50`); - const pageRes = await query.json(); + for (let startAt = 0; startAt < total; startAt += pageSize) { + if (startAt > 0 && pageDelayMs > 0) { + await delay(pageDelayMs); + } + console.log(`Fetching JIRA issues ${startAt + 1}-${Math.min(startAt + pageSize, total)} of ${total}...`); + const pageRes = await fetchJiraJson(`${baseUrl + encodedJql}&startAt=${startAt}&maxResults=${pageSize}&fields=${fields}`, maxRetries); pageRes.issues.forEach((issue) => { let parent = ''; @@ -38,7 +94,7 @@ export default async function fetchReleaseNotes(version) { priority: issue.fields.priority.id, component: issue.fields.components[0].name, subcomponent: `${issue.fields.components[0].name}${issue.fields.customfield_10008?.name ? `/${issue.fields.customfield_10008?.name}` : ''}`, - link: `https://bugs.openjdk.java.net/browse/${issue.key}`, + link: `https://bugs.openjdk.org/browse/${issue.key}`, type: issue.fields.issuetype.name, backportOf: parent || null, }); diff --git a/generate-release-notes/generate-release-notes/test/extractJDKIssuesTest.js b/generate-release-notes/generate-release-notes/test/extractJDKIssuesTest.js index 4eb040e..c4da5b6 100644 --- a/generate-release-notes/generate-release-notes/test/extractJDKIssuesTest.js +++ b/generate-release-notes/generate-release-notes/test/extractJDKIssuesTest.js @@ -1,7 +1,7 @@ import assert from 'assert'; import { test } from 'node:test'; import extractJDKIssues from '../lib/extractJDKIssues.js'; -import mockAPIResponse from './mock/mockAPIResponse.json' assert { type: 'json' }; +import mockAPIResponse from './mock/mockAPIResponse.json' with { type: 'json' }; test('extractJDKIssues', async (t) => { diff --git a/generate-release-notes/generate-release-notes/test/fetchGitHubCommitsTest.js b/generate-release-notes/generate-release-notes/test/fetchGitHubCommitsTest.js index 42afe90..4552a69 100644 --- a/generate-release-notes/generate-release-notes/test/fetchGitHubCommitsTest.js +++ b/generate-release-notes/generate-release-notes/test/fetchGitHubCommitsTest.js @@ -1,14 +1,14 @@ import assert from 'node:assert/strict'; import { test } from 'node:test'; import { fetchCommits } from '../lib/fetchGitHubCommits.js'; -import mockAPIResponse from './mock/mockAPIResponse.json' assert { type: 'json' }; +import mockAPIResponse from './mock/mockAPIResponse.json' with { type: 'json' }; const repository = 'adoptium/jdk17u' const baseTag = 'jdk-17.0.5-ga'; const tag = 'jdk-17.0.6+9'; test('fetchGithubCommits', async (t) => { - const commits = await fetchCommits( { repository, baseTag, tag }); + const commits = await fetchCommits({ repository, baseTag, tag }); await t.test('returns correct number of commits', (t) => { assert.strictEqual(commits.length, mockAPIResponse.total_commits); @@ -18,4 +18,51 @@ test('fetchGithubCommits', async (t) => { assert.strictEqual(commits[0].sha, mockAPIResponse.commits[0].sha); assert.strictEqual(commits[0].commit.message, mockAPIResponse.commits[0].commit.message); }); +}); + +test('fetchGithubCommits sends Authorization header when GITHUB_TOKEN is set', async () => { + const originalFetch = globalThis.fetch; + let capturedHeaders; + + globalThis.fetch = async (url, init) => { + capturedHeaders = init?.headers ?? {}; + return { + ok: true, + headers: { get: () => null }, + json: async () => ({ commits: [mockAPIResponse.commits[0]] }), + }; + }; + + process.env.GITHUB_TOKEN = 'test-token'; + + try { + await fetchCommits({ repository, baseTag, tag }); + assert.strictEqual(capturedHeaders.Authorization, 'Bearer test-token'); + } finally { + globalThis.fetch = originalFetch; + delete process.env.GITHUB_TOKEN; + } +}); + +test('fetchGithubCommits omits Authorization header when GITHUB_TOKEN is not set', async () => { + const originalFetch = globalThis.fetch; + let capturedHeaders; + + globalThis.fetch = async (url, init) => { + capturedHeaders = init?.headers ?? {}; + return { + ok: true, + headers: { get: () => null }, + json: async () => ({ commits: [mockAPIResponse.commits[0]] }), + }; + }; + + delete process.env.GITHUB_TOKEN; + + try { + await fetchCommits({ repository, baseTag, tag }); + assert.strictEqual(capturedHeaders.Authorization, undefined); + } finally { + globalThis.fetch = originalFetch; + } }); \ No newline at end of file diff --git a/generate-release-notes/generate-release-notes/test/fetchJiraIssuesTest.js b/generate-release-notes/generate-release-notes/test/fetchJiraIssuesTest.js index ed647e0..4048a88 100644 --- a/generate-release-notes/generate-release-notes/test/fetchJiraIssuesTest.js +++ b/generate-release-notes/generate-release-notes/test/fetchJiraIssuesTest.js @@ -2,29 +2,110 @@ import assert from 'node:assert/strict'; import { test } from 'node:test'; import fetchJiraIssues from '../lib/fetchJiraIssues.js'; -test('fetchReleaseNotes', async (t) => { - const version = '17.0.6'; +const mockIssue = { + key: 'JDK-8280124', + fields: { + summary: 'Reduce branches decoding latin-1 chars', + priority: { id: '3' }, + components: [{ name: 'core-libs' }], + customfield_10008: { name: 'java.lang' }, + issuetype: { name: 'Backport' }, + issuelinks: [ + { + type: { name: 'Backport' }, + inwardIssue: { key: 'JDK-8280123' }, + }, + ], + }, +}; - const issues = await fetchJiraIssues(version); +test('fetchReleaseNotes', async (t) => { + const originalFetch = globalThis.fetch; - await t.test('returns an array', () => { - assert.ok(Array.isArray(issues), 'should return an array'); + globalThis.fetch = async (url) => ({ + ok: true, + status: 200, + headers: { get: () => 'application/json;charset=UTF-8' }, + json: async () => { + if (url.includes('startAt=0') && url.includes('maxResults=1&')) { + return { total: 1 }; + } + if (url.includes('startAt=0')) { + return { total: 1, issues: [mockIssue] }; + } + return { total: 1, issues: [] }; + }, }); - await t.test('first issue extracts expected values', () => { - if (issues.length > 0) { + try { + const version = '17.0.6'; + const issues = await fetchJiraIssues(version); + + await t.test('returns an array', () => { + assert.ok(Array.isArray(issues), 'should return an array'); + assert.strictEqual(issues.length, 1); + }); + + await t.test('first issue extracts expected values', () => { const firstIssue = issues[0]; - assert.ok(typeof firstIssue.id === 'string', 'issue should have an id'); - assert.ok(typeof firstIssue.title === 'string', 'issue should have a title'); - assert.ok(typeof firstIssue.priority === 'string', 'issue should have a priority'); - assert.ok(typeof firstIssue.component === 'string', 'issue should have a component'); - assert.ok(typeof firstIssue.subcomponent === 'string', 'issue should have a subcomponent'); - assert.ok(typeof firstIssue.link === 'string', 'issue should have a link'); - assert.ok(typeof firstIssue.type === 'string', 'issue should have a type'); - assert.ok(typeof firstIssue.backportOf === 'string' || firstIssue.backportOf === null, 'issue should have a backportOf value'); - } else { - assert.fail('no issues were returned'); - } + assert.strictEqual(firstIssue.id, 'JDK-8280124'); + assert.strictEqual(firstIssue.title, 'Reduce branches decoding latin-1 chars'); + assert.strictEqual(firstIssue.priority, '3'); + assert.strictEqual(firstIssue.component, 'core-libs'); + assert.strictEqual(firstIssue.subcomponent, 'core-libs/java.lang'); + assert.strictEqual(firstIssue.link, 'https://bugs.openjdk.org/browse/JDK-8280124'); + assert.strictEqual(firstIssue.type, 'Backport'); + assert.strictEqual(firstIssue.backportOf, 'JDK-8280123'); + }); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test('fetchJiraIssues throws a clear error on HTTP error response', async () => { + const originalFetch = globalThis.fetch; + + globalThis.fetch = async () => ({ + ok: false, + status: 400, + headers: { get: () => 'text/html' }, }); + + try { + await assert.rejects( + () => fetchJiraIssues('17.0.6'), + (err) => { + assert.ok(err.message.includes('400'), `Expected 400 in error message, got: ${err.message}`); + return true; + }, + ); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test('fetchJiraIssues throws a clear error when response is HTML not JSON', async () => { + const originalFetch = globalThis.fetch; + + globalThis.fetch = async () => ({ + ok: true, + status: 200, + headers: { get: () => 'text/html; charset=utf-8' }, + text: async () => 'Service Unavailable', + json: async () => { throw new SyntaxError('Unexpected token <'); }, + }); + + try { + await assert.rejects( + () => fetchJiraIssues('17.0.6', 1), + (err) => { + assert.ok(err.message.includes('non-JSON'), `Expected "non-JSON" in error message, got: ${err.message}`); + assert.ok(err.message.includes('text/html'), `Expected content-type in error message, got: ${err.message}`); + return true; + }, + ); + } finally { + globalThis.fetch = originalFetch; + } });