Fix fetchGitHubCommits to use authenticated scrapes via username/PAT

generate-release-notes/generate-release-notes/lib/fetchGitHubCommits.js

@@ -9,11 +9,13 @@ export const fetchCommits = async ({
   let nextPage = true;
   let commits = [];
 
-  console.log(`Fetching commits for ${repository} between ${baseTag} and ${tag}...`);
+  const token = process.env.GITHUB_TOKEN;
+  const headers = token ? { Authorization: `Bearer ${token}` } : {};
+  console.log(`Fetching commits for ${repository} between ${baseTag} and ${tag} (${token ? 'authenticated' : 'unauthenticated'})...`);
   while (nextPage) {
     const githubQuery = `https://api.github.com/repos/${repository}/compare/${baseTag}...${tag}?per_page=${pageSize}&page=${page}`;
     console.log(`Fetching commits from ${githubQuery}`);
-    const githubResponse = await fetch(githubQuery);
+    const githubResponse = await fetch(githubQuery, { headers });
     if (!githubResponse.ok) {
       throw new Error(`Failed to fetch commits from ${githubQuery}, status: ${githubResponse.status}`);
     }

generate-release-notes/generate-release-notes/lib/fetchJiraIssues.js

@@ -1,22 +1,78 @@
 // Description: Fetches the JIRA issues from bugs.openjdk.org
-export default async function fetchReleaseNotes(version) {
+
+const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+
+async function fetchJiraJson(url, maxRetries = 5) {
+  const headers = {
+    Accept: 'application/json',
+    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36',
+  };
+
+  for (let attempt = 1; attempt <= maxRetries; attempt += 1) {
+    try {
+      const response = await fetch(url, { headers, signal: AbortSignal.timeout(60000) });
+      if (!response.ok) {
+        if (response.status === 429 || response.status >= 500) {
+          if (attempt < maxRetries) {
+            const backoff = Math.min(attempt * 10000, 60000);
+            console.warn(`Jira API HTTP ${response.status}. Retrying in ${backoff / 1000}s (attempt ${attempt}/${maxRetries})...`);
+            await delay(backoff);
+            continue;
+          }
+        }
+        throw new Error(`Jira API request failed: HTTP ${response.status} for ${url}`);
+      }
+      const contentType = response.headers.get('content-type') || '';
+      if (!contentType.includes('application/json')) {
+        const body = await response.text();
+        if (attempt < maxRetries) {
+          const backoff = Math.min(attempt * 10000, 60000);
+          console.warn(`Jira API returned non-JSON response (content-type: ${contentType}). Retrying in ${backoff / 1000}s (attempt ${attempt}/${maxRetries})...`);
+          await delay(backoff);
+          continue;
+        }
+        throw new Error(`Jira API returned non-JSON response (content-type: ${contentType}) for ${url}. Body starts with: ${body.slice(0, 200)}`);
+      }
+      return await response.json();
+    } catch (err) {
+      if (attempt < maxRetries && (err.name === 'TimeoutError' || err.name === 'AbortError' || err.code === 'UND_ERR_CONNECT_TIMEOUT')) {
+        const backoff = Math.min(attempt * 10000, 60000);
+        console.warn(`Jira API request timed out. Retrying in ${backoff / 1000}s (attempt ${attempt}/${maxRetries})...`);
+        await delay(backoff);
+        continue;
+      }
+      throw err;
+    }
+  }
+}
+
+export default async function fetchReleaseNotes(version, maxRetries = 5) {
   // fetch the release notes from the bugs.openjdk.org
-  const baseUrl = 'https://bugs.openjdk.java.net/rest/api/2/search?jql=';
+  const baseUrl = 'https://bugs.openjdk.org/rest/api/2/search?jql=';
   const jql = `project=JDK AND (status in (Closed, Resolved))
     AND (resolution not in ("Won't Fix", "Duplicate", "Cannot Reproduce", "Not an Issue", "Withdrawn"))
     AND (labels not in (release-note, openjdk-na) OR labels is EMPTY)
     AND (summary !~ "release note") AND (issuetype != CSR) AND (fixVersion in (${version}))`;
+  const encodedJql = encodeURIComponent(jql);
+  const fields = 'summary,priority,components,customfield_10008,issuetype,issuelinks';
+  const pageSize = 100;
+  const pageDelayMs = 2500;
+
   // execute the initial fetch to get the total number of issues
-  const totalQuery = await fetch(`${baseUrl + jql}&startAt=1&maxResults=1`);
-  const initialRes = await totalQuery.json();
+  const initialRes = await fetchJiraJson(`${baseUrl + encodedJql}&startAt=0&maxResults=1&fields=summary`, maxRetries);
   const { total } = initialRes;
 
+  console.log(`Found ${total} JIRA issues for JDK ${version}. Fetching in pages of ${pageSize}...`);
+
   const JIRA_ISSUES = [];
 
   // fetch all the issues by page
-  for (let startAt = 0; startAt <= total + 50; startAt += 50) {
-    const query = await fetch(`${baseUrl + jql}&startAt=${startAt}&maxResults=50`);
-    const pageRes = await query.json();
+  for (let startAt = 0; startAt < total; startAt += pageSize) {
+    if (startAt > 0 && pageDelayMs > 0) {
+      await delay(pageDelayMs);
+    }
+    console.log(`Fetching JIRA issues ${startAt + 1}-${Math.min(startAt + pageSize, total)} of ${total}...`);
+    const pageRes = await fetchJiraJson(`${baseUrl + encodedJql}&startAt=${startAt}&maxResults=${pageSize}&fields=${fields}`, maxRetries);
 
     pageRes.issues.forEach((issue) => {
       let parent = '';
@@ -38,7 +94,7 @@ export default async function fetchReleaseNotes(version) {
         priority: issue.fields.priority.id,
         component: issue.fields.components[0].name,
         subcomponent: `${issue.fields.components[0].name}${issue.fields.customfield_10008?.name ? `/${issue.fields.customfield_10008?.name}` : ''}`,
-        link: `https://bugs.openjdk.java.net/browse/${issue.key}`,
+        link: `https://bugs.openjdk.org/browse/${issue.key}`,
         type: issue.fields.issuetype.name,
         backportOf: parent || null,
       });

generate-release-notes/generate-release-notes/test/extractJDKIssuesTest.js

@@ -1,7 +1,7 @@
 import assert from 'assert';
 import { test } from 'node:test';
 import extractJDKIssues from '../lib/extractJDKIssues.js';
-import mockAPIResponse from './mock/mockAPIResponse.json' assert { type: 'json' };
+import mockAPIResponse from './mock/mockAPIResponse.json' with { type: 'json' };
 
 
 test('extractJDKIssues', async (t) => {

generate-release-notes/generate-release-notes/test/fetchGitHubCommitsTest.js

@@ -1,14 +1,14 @@
 import assert from 'node:assert/strict';
 import { test } from 'node:test';
 import { fetchCommits } from '../lib/fetchGitHubCommits.js';
-import mockAPIResponse from './mock/mockAPIResponse.json' assert { type: 'json' };
+import mockAPIResponse from './mock/mockAPIResponse.json' with { type: 'json' };
 
 const repository = 'adoptium/jdk17u'
 const baseTag = 'jdk-17.0.5-ga';
 const tag = 'jdk-17.0.6+9';
 
 test('fetchGithubCommits', async (t) => {
-  const commits = await fetchCommits( { repository, baseTag, tag });
+  const commits = await fetchCommits({ repository, baseTag, tag });
 
   await t.test('returns correct number of commits', (t) => {
     assert.strictEqual(commits.length, mockAPIResponse.total_commits);
@@ -18,4 +18,51 @@ test('fetchGithubCommits', async (t) => {
     assert.strictEqual(commits[0].sha, mockAPIResponse.commits[0].sha);
     assert.strictEqual(commits[0].commit.message, mockAPIResponse.commits[0].commit.message);
   });
+});
+
+test('fetchGithubCommits sends Authorization header when GITHUB_TOKEN is set', async () => {
+  const originalFetch = globalThis.fetch;
+  let capturedHeaders;
+
+  globalThis.fetch = async (url, init) => {
+    capturedHeaders = init?.headers ?? {};
+    return {
+      ok: true,
+      headers: { get: () => null },
+      json: async () => ({ commits: [mockAPIResponse.commits[0]] }),
+    };
+  };
+
+  process.env.GITHUB_TOKEN = 'test-token';
+
+  try {
+    await fetchCommits({ repository, baseTag, tag });
+    assert.strictEqual(capturedHeaders.Authorization, 'Bearer test-token');
+  } finally {
+    globalThis.fetch = originalFetch;
+    delete process.env.GITHUB_TOKEN;
+  }
+});
+
+test('fetchGithubCommits omits Authorization header when GITHUB_TOKEN is not set', async () => {
+  const originalFetch = globalThis.fetch;
+  let capturedHeaders;
+
+  globalThis.fetch = async (url, init) => {
+    capturedHeaders = init?.headers ?? {};
+    return {
+      ok: true,
+      headers: { get: () => null },
+      json: async () => ({ commits: [mockAPIResponse.commits[0]] }),
+    };
+  };
+
+  delete process.env.GITHUB_TOKEN;
+
+  try {
+    await fetchCommits({ repository, baseTag, tag });
+    assert.strictEqual(capturedHeaders.Authorization, undefined);
+  } finally {
+    globalThis.fetch = originalFetch;
+  }
 });

generate-release-notes/generate-release-notes/test/fetchJiraIssuesTest.js

@@ -2,29 +2,110 @@ import assert from 'node:assert/strict';
 import { test } from 'node:test';
 import fetchJiraIssues from '../lib/fetchJiraIssues.js';
 
-test('fetchReleaseNotes', async (t) => {
-  const version = '17.0.6';
+const mockIssue = {
+  key: 'JDK-8280124',
+  fields: {
+    summary: 'Reduce branches decoding latin-1 chars',
+    priority: { id: '3' },
+    components: [{ name: 'core-libs' }],
+    customfield_10008: { name: 'java.lang' },
+    issuetype: { name: 'Backport' },
+    issuelinks: [
+      {
+        type: { name: 'Backport' },
+        inwardIssue: { key: 'JDK-8280123' },
+      },
+    ],
+  },
+};
 
-  const issues = await fetchJiraIssues(version);
+test('fetchReleaseNotes', async (t) => {
+  const originalFetch = globalThis.fetch;
 
-  await t.test('returns an array', () => {
-    assert.ok(Array.isArray(issues), 'should return an array');
+  globalThis.fetch = async (url) => ({
+    ok: true,
+    status: 200,
+    headers: { get: () => 'application/json;charset=UTF-8' },
+    json: async () => {
+      if (url.includes('startAt=0') && url.includes('maxResults=1&')) {
+        return { total: 1 };
+      }
+      if (url.includes('startAt=0')) {
+        return { total: 1, issues: [mockIssue] };
+      }
+      return { total: 1, issues: [] };
+    },
   });
 
-  await t.test('first issue extracts expected values', () => {
-    if (issues.length > 0) {
+  try {
+    const version = '17.0.6';
+    const issues = await fetchJiraIssues(version);
+
+    await t.test('returns an array', () => {
+      assert.ok(Array.isArray(issues), 'should return an array');
+      assert.strictEqual(issues.length, 1);
+    });
+
+    await t.test('first issue extracts expected values', () => {
       const firstIssue = issues[0];
 
-      assert.ok(typeof firstIssue.id === 'string', 'issue should have an id');
-      assert.ok(typeof firstIssue.title === 'string', 'issue should have a title');
-      assert.ok(typeof firstIssue.priority === 'string', 'issue should have a priority');
-      assert.ok(typeof firstIssue.component === 'string', 'issue should have a component');
-      assert.ok(typeof firstIssue.subcomponent === 'string', 'issue should have a subcomponent');
-      assert.ok(typeof firstIssue.link === 'string', 'issue should have a link');
-      assert.ok(typeof firstIssue.type === 'string', 'issue should have a type');
-      assert.ok(typeof firstIssue.backportOf === 'string' || firstIssue.backportOf === null, 'issue should have a backportOf value');
-    } else {
-      assert.fail('no issues were returned');
-    }
+      assert.strictEqual(firstIssue.id, 'JDK-8280124');
+      assert.strictEqual(firstIssue.title, 'Reduce branches decoding latin-1 chars');
+      assert.strictEqual(firstIssue.priority, '3');
+      assert.strictEqual(firstIssue.component, 'core-libs');
+      assert.strictEqual(firstIssue.subcomponent, 'core-libs/java.lang');
+      assert.strictEqual(firstIssue.link, 'https://bugs.openjdk.org/browse/JDK-8280124');
+      assert.strictEqual(firstIssue.type, 'Backport');
+      assert.strictEqual(firstIssue.backportOf, 'JDK-8280123');
+    });
+  } finally {
+    globalThis.fetch = originalFetch;
+  }
+});
+
+test('fetchJiraIssues throws a clear error on HTTP error response', async () => {
+  const originalFetch = globalThis.fetch;
+
+  globalThis.fetch = async () => ({
+    ok: false,
+    status: 400,
+    headers: { get: () => 'text/html' },
   });
+
+  try {
+    await assert.rejects(
+      () => fetchJiraIssues('17.0.6'),
+      (err) => {
+        assert.ok(err.message.includes('400'), `Expected 400 in error message, got: ${err.message}`);
+        return true;
+      },
+    );
+  } finally {
+    globalThis.fetch = originalFetch;
+  }
+});
+
+test('fetchJiraIssues throws a clear error when response is HTML not JSON', async () => {
+  const originalFetch = globalThis.fetch;
+
+  globalThis.fetch = async () => ({
+    ok: true,
+    status: 200,
+    headers: { get: () => 'text/html; charset=utf-8' },
+    text: async () => '<html><body>Service Unavailable</body></html>',
+    json: async () => { throw new SyntaxError('Unexpected token <'); },
+  });
+
+  try {
+    await assert.rejects(
+      () => fetchJiraIssues('17.0.6', 1),
+      (err) => {
+        assert.ok(err.message.includes('non-JSON'), `Expected "non-JSON" in error message, got: ${err.message}`);
+        assert.ok(err.message.includes('text/html'), `Expected content-type in error message, got: ${err.message}`);
+        return true;
+      },
+    );
+  } finally {
+    globalThis.fetch = originalFetch;
+  }
 });