GH-51098: [Release] Update CHANGELOG.md as a post release task

.github/workflows/dev.yml

@@ -102,7 +102,14 @@ jobs:
         shell: bash
         run: |
           gem install test-unit openssl
-          pip install build "cython>=3.1" pytest requests scikit-build-core setuptools-scm
+          pip install \
+            build \
+            "cython>=3.1" \
+            "dev/archery[release]" \
+            pytest \
+            requests \
+            scikit-build-core \
+            setuptools-scm
       - name: Run Release Test
         shell: bash
         run: |

dev/archery/MANIFEST.in

@@ -1,4 +1,4 @@
 include ../../LICENSE.txt
 include ../../NOTICE.txt
 
-include archery/reports/*
+include archery/templates/*

dev/archery/archery/release/cli.py

@@ -67,13 +67,14 @@ def release_changelog_add(obj, version):
 
     # just handle the current version
     release = Release(version, repo=repo, issue_tracker=issue_tracker)
-    if release.is_released:
-        raise ValueError('This version has been already released!')
 
     changelog = release.changelog()
     changelog_path = pathlib.Path(repo) / 'CHANGELOG.md'
 
     current_content = changelog_path.read_text()
+    if f'# Apache Arrow {version} (' in current_content:
+        raise ValueError(
+            f'CHANGELOG.md already contains the changelog of {version}!')
     new_content = changelog.render('markdown') + current_content
 
     changelog_path.write_text(new_content)

dev/archery/archery/release/core.py

@@ -136,6 +136,11 @@ def project_issues(self, version):
         issues = self.github_repo.get_issues(
             milestone=self._milestone_from_semver(version),
             state="all")
+        # This is only for testing. We can limit the number of issues
+        # to be processed for faster testing.
+        max_issues = os.environ.get("ARCHERY_MAX_PROJECT_ISSUES")
+        if max_issues is not None:
+            issues = issues[:int(max_issues)]
         return list(map(Issue.from_github, issues))
 
     def issue(self, key):

dev/release/post-10-bump-versions-test.rb

@@ -20,6 +20,8 @@ class PostBumpVersionsTest < Test::Unit::TestCase
   include VersionDetectable
 
   def setup
+    @env = File.expand_path("dev/release/.env")
+
     @current_commit = git_current_commit
     detect_versions
 
@@ -480,4 +482,53 @@ def test_linux_packages
                  parse_patch(git("log", "-n", "1", "-p")),
                  "Output:\n#{stdout}")
   end
+
+  def normalized_time
+    "1970-01-01 00:00:00+00:00"
+  end
+
+  def normalize_time(string)
+    string.gsub(/\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}[+-]\d{2}:\d{2}/) do
+      normalized_time
+    end
+  end
+
+  def test_changelog
+    omit_on_release_branch
+
+    github_token = File.read(@env)[/^GH_TOKEN=(.*)$/, 1]
+    stdout = bump_versions("CHANGELOG",
+                           # Don't process all issues for faster testing
+                           "ARCHERY_MAX_PROJECT_ISSUES" => "10",
+                           "GH_TOKEN" => github_token)
+    changes = parse_patch(git("log", "-n", "1", "-p"))
+    sampled_changes = changes.collect do |change|
+      sampled_hunks = change[:hunks].collect do |hunk|
+        first_added_line = hunk.find {|line| line.start_with?("+")}
+        first_added_line = normalize_time(first_added_line)
+        {
+          first_added_line: first_added_line,
+          first_removed_line: hunk.find {|line| line.start_with?("-")},
+        }
+      end
+      {
+        sampled_hunks: sampled_hunks,
+        path: change[:path],
+      }
+    end
+    expected_changes = [
+      {
+        sampled_hunks: [
+          {
+            first_added_line: "+# Apache Arrow #{released_version} (#{normalized_time})",
+            first_removed_line: nil,
+          },
+        ],
+        path: "CHANGELOG.md",
+      },
+    ]
+    assert_equal(expected_changes,
+                 sampled_changes,
+                 "Output:\n#{stdout}")
+  end
 end

dev/release/post-10-bump-versions.sh

@@ -29,6 +29,7 @@ fi
 
 : ${BUMP_DEFAULT:=1}
 : ${BUMP_UPDATE_LOCAL_DEFAULT_BRANCH:=${BUMP_DEFAULT}}
+: ${BUMP_CHANGELOG:=${BUMP_DEFAULT}}
 : ${BUMP_VERSION_POST_TAG:=${BUMP_DEFAULT}}
 : ${BUMP_DEB_PACKAGE_NAMES:=${BUMP_DEFAULT}}
 : ${BUMP_LINUX_PACKAGES:=${BUMP_DEFAULT}}
@@ -70,6 +71,13 @@ if [ ${BUMP_UPDATE_LOCAL_DEFAULT_BRANCH} -gt 0 ]; then
   git rebase upstream/${DEFAULT_BRANCH}
 fi
 
+if [ ${BUMP_CHANGELOG} -gt 0 ]; then
+  echo "Updating CHANGELOG.md for ${version}"
+  archery release --src "${ARROW_DIR}" changelog add "${version}"
+  git add "${ARROW_DIR}/CHANGELOG.md"
+  git commit -m "MINOR: [Release] Update CHANGELOG.md for ${version}"
+fi
+
 if [ ${BUMP_VERSION_POST_TAG} -gt 0 ]; then
   echo "Updating versions for ${next_version_snapshot}"
   update_versions "${version}" "${next_version}" "snapshot"