rssharvester.py

#import libraries
import feedparser
import csv
import re
import time
import html2text
from humanfriendly import format_timespan

#configure html2text
textmaker = html2text.HTML2Text()
textmaker.body_width=0  # disable line wrapping
textmaker.ignore_links=False
textmaker.ignore_emphasis=True
textmaker.strong_mark=''
textmaker.use_automatic_links=True

#print explanatory text and instructions
print('-----')
print('Welcome!  This script is written to parse RSS feeds for podcasts and convert selected metadata elements into a CSV spreadsheet.  This script was written at the University of Minnesota, so the output formatted for ingesting into our Institutional Repository, but may have uses well beyond that.  Formatting is more or less basic Dublic Core metadata in a CSV file, with some extra DSpace-specific fields added.')
print()
print("You can run this script for a single podcast, or for several podcasts at once.  To run on a single podcast, you'll need the URL for the podcast's RSS feed.  You'll also need to provide a title for the resulting CSV. To run this script for several podcasts at once, you'll need a CSV file with the RSS URL in Column A, and the desired title for the output log in Column B.  There should be no header rows.")
print()


#determine starting input (single URL or csv file list)
print('-----')
styleinput=input('Would you like to run this script by providing a single RSS URL, or would you like to run it on a CSV with RSS URLs?  Enter "1" for a single URL or "2" for a multi-URL csv: ')
print()
if styleinput in ('1','one','One','ONE'):
    #get starting RSS URL from user, establish output filename
    inname=input('Enter RSS URL: ')
    outname=input('Enter the desired filename for the output CSV: ')
    feedlist={inname:outname}
    rowcounter=1
elif styleinput in ('2','two','Two','TWO'):
    #get filepath to CSV from user, establish list of input URLs and output filenames
    csv_input = input("Enter the filepath and name of the input CSV: ")
    feedlist={}
    with open(csv_input, mode='r', encoding="utf8", errors='ignore') as userinput:
        reader = csv.DictReader(userinput)
        rowcounter=0
        for row in csv.reader(userinput):
            inname=row[0]
            outname=row[1]
            feedlist[inname]=outname
            rowcounter=rowcounter+1
else:
    print('-----')
    print('PROBLEM WITH YOUR INPUT!  Re-run this script, and be sure to enter "1" to run this script on a single RSS URL or "2" to start with a CSV list.')
    quit()

#show user number of feeds to be parsed, confirm before proceeding
print('-----')
print('This script is about to parse ',rowcounter,' RSS feeds.')  
proceedinput=input('Are you ready to proceed? Enter "Y" to proceed or "N" to close this script: ')
if proceedinput in ('y','Y','yes','Yes','YES'):
    pass
elif proceedinput in ('n','N','no','No','NO'):
    quit()
else:
    print('-----')
    print('PROBLEM WITH YOUR INPUT!  Re-run this script, and be sure to enter "Y" to run this script on the specified number of URLs or "N" to quit.')
    quit()

print('-----')
print('Processing...')
print()

for key in feedlist:
    logname=feedlist[key]
    feed=feedparser.parse(key)
    
    #skip extraction if no entries found
    if len(feed['entries'])==0:
        print('-----')
        print('There was an unexpected problem pulling data from ',inname,'. This URL has been skipped.')
        print('Processing...')
        print()
        continue
    else:
        #set up CSV file and header row
        import csv
        with open(logname, mode='w', newline='', encoding='utf-8') as csvfile:
            writer = csv.writer(csvfile, delimiter=',')
            writer.writerow(['bundle:ORIGINAL', 'bundle:THUMBNAIL', 'bundle:TEXT','dc.title', 'dc.date.issued', 'dc.description.abstract', 'dc.description', 'dc.type.','dc.contributor.author'])
        
        #get default image value for podcast
        try:
            defaultthumb=feed.feed.image['href']
        except:
            defaultthumb="No Image Available"
        
        #validating each entry in RSS and converting into variables
        for entry in feed.entries:
            #title
            try:
                title=entry.title
            except:
                title="No Title Available"

            #date
            try:
                parseddate=entry.published_parsed
                date=time.strftime('%Y-%m-%d',parseddate)
            except:
                date="No Date Available"

            #episode summary
            try:
                htmlabstract=entry.description
                rawabstract=textmaker.handle(htmlabstract)
                mezzoabstract=rawabstract.replace("[","")
                abstract=mezzoabstract.replace("]("," (")
            except:
                abstract="No Abstract Available"

            #runtime
            try:
                duration=entry.itunes_duration
                try:
                  seconds=int(duration)
                  try:
                    runtime=format_timespan(seconds)
                    description="Runtime: "+runtime
                  except:
                    description=str(seconds)+" seconds"
                except:
                  description=duration
            except:
                description="No Runtime Available"

            #audio files
            try:
                files=entry.enclosures[0]['href']
            except:
                files="No Files Available"

            #episode thumbnail
            try:
                image=entry.image['href']
            except:
                try:
                    image=defaultthumb
                except:
                    image="No Image Available"

            #transcript
            try:
              transcript=entry.podcast_transcript['url']
            except:
              transcript="No Transcript Available"

            #item type
            type="Audio"

        
        #write entry metadata into a new row
            entrydata=[files,image, transcript, title, date, abstract, description, type]
            with open(logname, mode='a', newline='', encoding='utf-8') as csvfile:
                writer = csv.writer(csvfile, delimiter=',')
                writer.writerow(entrydata)

print()
print('Process is completed!')