Finding 'A Good Read' with Ollama and Llama 3

from bs4 import BeautifulSoup
import requests
import ollama
import textwrap
import tqdm
import json
import time
import pandas as pd
from itertools import chain

def find_episodes(programme_id):
    """
    For a programme on BBC iPlayer with a URL:
    https://www.bbc.co.uk/programmes/b006v8jn/episodes/player?page=1
    the ID is "b006v8jn"
    """
    page = 0
    urls_list = []
    while True:
        page += 1
        url = f"https://www.bbc.co.uk/programmes/{programme_id}/episodes/player?page={page}"
        response = requests.get(url)
        if response.status_code == 200:
            html = response.content.decode("UTF-8")
            soup = BeautifulSoup(html, 'html.parser')
            episodes = soup.find_all("div", class_="programme__body", recursive=True)
            if len(episodes) == 0:
                break
            else:
                for episode in episodes:
                    url = episode.h2.a.attrs['href']
                    urls_list.append(url)
        else:
            break
    return urls_list
    

urls_list = find_episodes("b006v8jn")

len(urls_list)

texts = []
for url in urls_list:
    response = requests.get(url)
    if response.status_code != 200:
        print("error")
    html = response.content.decode("UTF-8")
    soup = BeautifulSoup(html, 'html.parser')
    text = soup.find("div", class_="synopsis-toggle__long")
    if not text:
        text = soup.find("div", class_="longest-synopsis")
    if not text:
        print(f"skipping url {url}")
        break
    text = text.get_text().replace("\n", " ").lstrip().rstrip()
    texts.append(text)

texts[0]

"ABSENT IN THE SPRING by Agatha Christie (writing as Mary Westmacott) (HarperCollins), chosen by Simon BrettIN THE GARDEN OF THE FUGITIVES by Ceridwen Dovey (Penguin), chosen by Denise MinaHIDE MY EYES by Margery Allingham (Penguin), chosen by Harriett Gilbert  Crime writers Denise Mina and Simon Brett join Harriett Gilbert to read each other's favourite books.  Simon Brett (Charles Paris, Fethering and Mrs Pargeter detective series) chooses Agatha Christie under the pseudonym Mary Westmacott, with Absent In The Spring. It’s a story without any detective and one that, perhaps, reveals a more personal side to Christie's writing.  Denise Mina (most recently: Three Fires, The Second Murderer) picks In the Garden of the Fugitives by South African-Australian author Ceridwen Dovey, an epistolary novel which begins with a letter that breaks seventeen years of silence between a rich, elderly man with a broken heart and his former protegee, a young South African filmmaker.  And for the occasion of having two crime authors, Harriett Gilbert picks a golden age crime book, Hide My Eyes by Margery Allingham, where private detective Albert Campion finds himself hunting down a serial killer. Producer: Eliza Lomas for BBC Audio in Bristol Join the conversation @agoodreadbbc Instagram  Show less"

responses = []
for i, text in tqdm.tqdm(enumerate(texts)):
    response = json.loads(
        ollama.chat(
            model='llama3:8b-instruct-q4_0',
            messages=[
                {
                    'role': 'system',
                    'content': textwrap.dedent(
                        """\
                        You are a librarian.
                        You can read text from a description of a radio
                        programme very carefully and extract information about
                        books from it. You should not include information about
                        the people who are on the programme. You should only
                        extract information about the books they have
                        recommended. There are several books to extract from the
                        text, not just one. You only return the author and title
                        of books and no other information. 
                        If the book has been translated, do not include the 
                        translator.
                        
                        Your output should be a list of any detected books in
                        JSON like this:
                        
                        {"books": [
                            {"author": "JRR Tolkien",
                             "title": "Lord of the Rings"},
                            {"author": "JK Rowling",
                             "title": "Harry Potter and the Philosopher's "
                             "Stone"},
                        ]}

                        Fix any grammar issues. For e.g. ensure that everything
                        has the correct capitalisation.
                        """)
                },
                {
                    'role': 'user',
                    'content': text,
                },
            ], 
            format='json')['message']['content'])
    responses.append(response)

670it [27:39,  2.48s/it]

books = [r['books'] for r in responses]
books = list(chain.from_iterable(books))
len(books)

1986

f = open("books.json", 'w')
f.write(json.dumps(books))

books[:9]

[{'author': 'Agatha Christie', 'title': 'Absent In The Spring'},
     {'author': 'Ceridwen Dovey', 'title': 'In the Garden of the Fugitives'},
     {'author': 'Margery Allingham', 'title': 'Hide My Eyes'},
     {'author': 'Barbara Pym', 'title': 'Quartet in Autumn'},
     {'author': 'Rachel Ingalls', 'title': 'Mrs Caliban'},
     {'author': 'Derek Jarman', 'title': 'Pharmacopoeia: A Dungeness Notebook'},
     {'author': 'Julian Barnes', 'title': "Flaubert's Parrot"},
     {'author': 'John Higgs', 'title': 'The KLF'},
     {'author': 'Maggie Nelson', 'title': 'The Red Parts'}]

texts[:3]

["ABSENT IN THE SPRING by Agatha Christie (writing as Mary Westmacott) (HarperCollins), chosen by Simon BrettIN THE GARDEN OF THE FUGITIVES by Ceridwen Dovey (Penguin), chosen by Denise MinaHIDE MY EYES by Margery Allingham (Penguin), chosen by Harriett Gilbert  Crime writers Denise Mina and Simon Brett join Harriett Gilbert to read each other's favourite books.  Simon Brett (Charles Paris, Fethering and Mrs Pargeter detective series) chooses Agatha Christie under the pseudonym Mary Westmacott, with Absent In The Spring. It’s a story without any detective and one that, perhaps, reveals a more personal side to Christie's writing.  Denise Mina (most recently: Three Fires, The Second Murderer) picks In the Garden of the Fugitives by South African-Australian author Ceridwen Dovey, an epistolary novel which begins with a letter that breaks seventeen years of silence between a rich, elderly man with a broken heart and his former protegee, a young South African filmmaker.  And for the occasion of having two crime authors, Harriett Gilbert picks a golden age crime book, Hide My Eyes by Margery Allingham, where private detective Albert Campion finds himself hunting down a serial killer. Producer: Eliza Lomas for BBC Audio in Bristol Join the conversation @agoodreadbbc Instagram  Show less",
     "QUARTET IN AUTUMN by Barbara Pym, chosen by Samantha Harvey MRS CALIBAN by Rachel Ingalls, chosen by Harriett GilbertPHARMACOPOEIA: A DUNGENESS NOTEBOOK by Derek Jarman, chosen by Darran Anderson Two award-winning writers share books they love with Harriett Gilbert. Samantha Harvey is the author of five novels, The Wilderness, All Is Song, Dear Thief ,The Western Wind and, most recently, Orbital. She is also the author of a memoir, The Shapeless Unease: A Year of Not Sleeping. Her choice of a good read is a slim novel by Barbara Pym set in 1970s London about the lives of four single people in their sixties who work in an office together. Quartet in Autumn is sharply perceptive about the ways in which we hide from one other and from ourselves. Darran Anderson is an Irish writer who lives in London. He is the author of Imaginary Cities: A Tour of Dream Cities, Nightmare Cities, and Everywhere in Between; a memoir, Inventory, about growing up during the Troubles; and the forthcoming In the Land of My Enemy. His choice, Pharmacopoeia, brings together fragments of the artist and filmmaker Derek Jarman's writing on nature, gardening and Prospect Cottage, his Victorian fisherman's hut on the shingle at Dungeness.  Harriett's choice is a fantastically strange novel by Rachel Ingalls, published in 1982. In Mrs Caliban, a grieving housewife in a loveless marriage embarks on a heady affair with a green-skinned frogman.  Produced by Mair Bosworth for BBC Audio  Show less",
     "Historian and author Kathryn Hughes and No Such Thing As a Fish presenter Dan Schreiber recommend favourite books to Harriett Gilbert. Kathryn chooses Flaubert's Parrot by Julian Barnes, an exploration of the French writer's life in the form of a novel. Dan's choice is very different - John Higgs taking on the conceptual artists and chart toppers The KLF. Harriett has gone for Michael Ondaatje's novel Warlight, set in a murky and mysterious post-war London. Presenter: Harriett Gilbert Producer for BBC Audio Bristol: Sally Heaven  Show less"]

df = pd.DataFrame(books)
df

df.value_counts()[:25]

    author              title                         
    Angela Carter       Wise Children                     4
    Jean Rhys           Wide Sargasso Sea                 4
    Patrick Hamilton    Hangover Square                   4
    F Scott Fitzgerald  The Great Gatsby                  4
    Chinua Achebe       Things Fall Apart                 3
    Dodie Smith         I Capture The Castle              3
    Evelyn Waugh        A Handful of Dust                 3
    Joan Didion         The Year of Magical Thinking      3
    Sam Selvon          The Lonely Londoners              3
    Robert Graves       Goodbye to All That               3
    Evelyn Waugh        Scoop                             3
    Graham Swift        Waterland                         3
    Bernhard Schlink    The Reader                        3
    Flann O'Brien       The Third Policeman               3
    Graham Greene       Monsignor Quixote                 3
    Muriel Spark        The Girls of Slender Means        3
    Barbara Pym         Excellent Women                   3
    Lorna Sage          Bad Blood                         3
    Annie Proulx        The Shipping News                 3
    Italo Calvino       Invisible Cities                  3
    Michael Frayn       Towards the End of the Morning    3
    Tobias Wolff        Old School                        3
    Kingsley Amis       Lucky Jim                         3
    Jerome K Jerome     Three Men in a Boat               2
    Jane Austen         Pride and Prejudice               2
    Name: count, dtype: int64

print(df['author'].value_counts()[:25])

    author
    Graham Greene             15
    Evelyn Waugh              14
    Muriel Spark              12
    George Orwell             11
    John Steinbeck            10
    Michael Frayn              9
    Penelope Fitzgerald        9
    Angela Carter              9
    Ian McEwan                 9
    Vladimir Nabokov           8
    Anne Tyler                 8
    Patrick Hamilton           7
    Margaret Atwood            7
    Elizabeth Taylor           7
    Barbara Pym                7
    Philip Roth                7
    Henry James                6
    Gabriel Garcia Marquez     6
    Italo Calvino              6
    Kazuo Ishiguro             6
    Jean Rhys                  6
    Truman Capote              6
    Beryl Bainbridge           6
    PG Wodehouse               6
    Jane Austen                6
    Name: count, dtype: int64